diff --git a/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p25__forcebal/rep1/mem_samples.csv b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p25__forcebal/rep1/mem_samples.csv index b4d5f56b54d901670696f6928ff3e7fd68e2c28d..c6ae77ca35e42a7d202d72bf40ea5730b6bde77f 100644 --- a/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p25__forcebal/rep1/mem_samples.csv +++ b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p25__forcebal/rep1/mem_samples.csv @@ -62,3 +62,131 @@ 5, 1803 6, 1803 7, 1803 +0, 2351 +1, 2329 +2, 2329 +3, 2331 +4, 2319 +5, 2321 +6, 2331 +7, 2345 +0, 46703 +1, 46137 +2, 46137 +3, 46837 +4, 46837 +5, 46837 +6, 46837 +7, 46837 +0, 47107 +1, 46537 +2, 46537 +3, 47237 +4, 47237 +5, 47237 +6, 47237 +7, 47237 +0, 48395 +1, 47825 +2, 47825 +3, 48525 +4, 48525 +5, 48525 +6, 48525 +7, 48525 +0, 48675 +1, 48105 +2, 48105 +3, 48805 +4, 48805 +5, 48805 +6, 48805 +7, 48805 +0, 48675 +1, 48105 +2, 48105 +3, 48805 +4, 48805 +5, 48805 +6, 48805 +7, 48805 +0, 50879 +1, 50309 +2, 50309 +3, 51009 +4, 51009 +5, 51009 +6, 51009 +7, 51009 +0, 74329 +1, 74365 +2, 74345 +3, 74375 +4, 74375 +5, 74375 +6, 74375 +7, 74375 +0, 74975 +1, 75011 +2, 74991 +3, 75021 +4, 75021 +5, 75021 +6, 75021 +7, 75021 +0, 74977 +1, 75011 +2, 74991 +3, 75021 +4, 75021 +5, 75021 +6, 75021 +7, 75021 +0, 74981 +1, 75015 +2, 74995 +3, 75025 +4, 75025 +5, 75025 +6, 75025 +7, 75025 +0, 75555 +1, 75605 +2, 75571 +3, 75609 +4, 75633 +5, 75591 +6, 75583 +7, 75591 +0, 82245 +1, 82119 +2, 82135 +3, 82133 +4, 82127 +5, 82111 +6, 82193 +7, 82177 +0, 97625 +1, 97677 +2, 97675 +3, 97551 +4, 97551 +5, 97551 +6, 97551 +7, 97551 +0, 96795 +1, 97109 +2, 97427 +3, 97239 +4, 97041 +5, 96983 +6, 97239 +7, 96977 +0, 96727 +1, 3844 +2, 2564 +3, 2564 +4, 2564 +5, 2564 +6, 2564 +7, 2820 diff --git a/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p25__forcebal/rep1/train.log b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p25__forcebal/rep1/train.log index 43074f6ef06acb0f263ddd377ad9d43ad43998a2..0c7fd96884d1f7b5bbcfa56353022e09936f952c 100644 --- a/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p25__forcebal/rep1/train.log +++ b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p25__forcebal/rep1/train.log @@ -645,3 +645,5449 @@ building GPT model ... warnings.warn( /usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. warnings.warn( +[INFO] Use MNNVL fabric: False +[WARN] MAX_NUM_NVL_PEERS is not set. Using detected value 8. +NVSHMEM v3.7.2 +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +UltraEP: NVL domain size <= 8; forcing weight_sync plan mode to direct (was adaptive). + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 7262624256 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 22 +Params for bucket 1 (180289536 elements, 180289536 padded size): + module.output_layer.weight +Params for bucket 2 (73403904 elements, 73403904 padded size): + module.decoder.final_layernorm.weight + module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight +Params for bucket 3 (146800640 elements, 146800640 padded size): + module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight +Params for bucket 4 (103001600 elements, 103001600 padded size): + module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.9.self_attention.linear_qkv.weight + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.self_attention.linear_qkv.bias + module.decoder.layers.9.pre_mlp_layernorm.weight + module.decoder.layers.9.self_attention.linear_proj.weight + module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight +Params for bucket 5 (146800640 elements, 146800640 padded size): + module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight +Params for bucket 6 (103001600 elements, 103001600 padded size): + module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.8.self_attention.linear_qkv.weight + module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.pre_mlp_layernorm.weight + module.decoder.layers.8.self_attention.linear_qkv.bias + module.decoder.layers.8.self_attention.linear_proj.weight + module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight +Params for bucket 7 (146800640 elements, 146800640 padded size): + module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight +Params for bucket 8 (103001600 elements, 103001600 padded size): + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.7.self_attention.linear_qkv.bias + module.decoder.layers.7.pre_mlp_layernorm.weight + module.decoder.layers.7.self_attention.linear_proj.weight + module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.7.self_attention.linear_qkv.weight + module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight +Params for bucket 9 (146800640 elements, 146800640 padded size): + module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight +Params for bucket 10 (103001600 elements, 103001600 padded size): + module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.self_attention.linear_qkv.bias + module.decoder.layers.6.self_attention.linear_qkv.weight + module.decoder.layers.6.self_attention.linear_proj.weight + module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight +Params for bucket 11 (146800640 elements, 146800640 padded size): + module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight +Params for bucket 12 (103001600 elements, 103001600 padded size): + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.5.self_attention.linear_qkv.bias + module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.5.self_attention.linear_proj.weight + module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.5.self_attention.linear_qkv.weight + module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight +Params for bucket 13 (146800640 elements, 146800640 padded size): + module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight +Params for bucket 14 (103001600 elements, 103001600 padded size): + module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.4.self_attention.linear_qkv.weight + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.pre_mlp_layernorm.weight + module.decoder.layers.4.self_attention.linear_qkv.bias + module.decoder.layers.4.self_attention.linear_proj.weight + module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight +Params for bucket 15 (146800640 elements, 146800640 padded size): + module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight +Params for bucket 16 (103001600 elements, 103001600 padded size): + module.decoder.layers.3.self_attention.linear_qkv.weight + module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.pre_mlp_layernorm.weight + module.decoder.layers.3.self_attention.linear_qkv.bias + module.decoder.layers.3.self_attention.linear_proj.weight +Params for bucket 17 (146800640 elements, 146800640 padded size): + module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight +Params for bucket 18 (103001600 elements, 103001600 padded size): + module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.2.self_attention.linear_qkv.weight + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.2.self_attention.linear_qkv.bias + module.decoder.layers.2.self_attention.linear_proj.weight + module.decoder.layers.1.mlp.shared_experts.linear_fc2.weight +Params for bucket 19 (146800640 elements, 146800640 padded size): + module.decoder.layers.1.mlp.shared_experts.linear_fc1.weight +Params for bucket 20 (103001600 elements, 103001600 padded size): + module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.1.self_attention.linear_qkv.weight + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.1.self_attention.linear_qkv.bias + module.decoder.layers.1.self_attention.linear_proj.weight + module.decoder.layers.0.mlp.shared_experts.linear_fc2.weight +Params for bucket 21 (146800640 elements, 146800640 padded size): + module.decoder.layers.0.mlp.shared_experts.linear_fc1.weight +Params for bucket 22 (209890816 elements, 209890816 padded size): + module.decoder.layers.0.mlp.router.weight + module.decoder.layers.0.self_attention.linear_qkv.weight + module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.0.pre_mlp_layernorm.weight + module.decoder.layers.0.self_attention.linear_qkv.bias + module.decoder.layers.0.self_attention.linear_proj.weight + module.embedding.word_embeddings.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 44 +Params for bucket 1 (45875200 elements, 45875200 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight7 + module.decoder.layers.9.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc2.weight3 + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 +Params for bucket 2 (45875200 elements, 45875200 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight2 + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 + module.decoder.layers.9.mlp.experts.linear_fc1.weight7 + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 +Params for bucket 3 (55050240 elements, 55050240 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight6 + module.decoder.layers.9.mlp.experts.linear_fc1.weight5 + module.decoder.layers.9.mlp.experts.linear_fc1.weight4 +Params for bucket 4 (55050240 elements, 55050240 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 + module.decoder.layers.9.mlp.experts.linear_fc1.weight2 +Params for bucket 5 (45875200 elements, 45875200 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 + module.decoder.layers.8.mlp.experts.linear_fc2.weight5 + module.decoder.layers.8.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc2.weight7 +Params for bucket 6 (45875200 elements, 45875200 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 + module.decoder.layers.8.mlp.experts.linear_fc2.weight4 + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 +Params for bucket 7 (55050240 elements, 55050240 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight7 + module.decoder.layers.8.mlp.experts.linear_fc1.weight5 + module.decoder.layers.8.mlp.experts.linear_fc1.weight6 +Params for bucket 8 (55050240 elements, 55050240 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight4 + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 +Params for bucket 9 (45875200 elements, 45875200 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight1 + module.decoder.layers.7.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 +Params for bucket 10 (45875200 elements, 45875200 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 + module.decoder.layers.7.mlp.experts.linear_fc2.weight5 + module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.7.mlp.experts.linear_fc2.weight4 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 +Params for bucket 11 (55050240 elements, 55050240 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 + module.decoder.layers.7.mlp.experts.linear_fc1.weight6 + module.decoder.layers.7.mlp.experts.linear_fc1.weight7 + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 +Params for bucket 12 (55050240 elements, 55050240 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight5 + module.decoder.layers.7.mlp.experts.linear_fc1.weight4 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 +Params for bucket 13 (55050240 elements, 55050240 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 +Params for bucket 14 (45875200 elements, 45875200 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight3 + module.decoder.layers.6.mlp.experts.linear_fc2.weight5 +Params for bucket 15 (45875200 elements, 45875200 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 +Params for bucket 16 (55050240 elements, 55050240 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight6 + module.decoder.layers.6.mlp.experts.linear_fc1.weight5 + module.decoder.layers.6.mlp.experts.linear_fc1.weight4 +Params for bucket 17 (55050240 elements, 55050240 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight3 + module.decoder.layers.6.mlp.experts.linear_fc1.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 +Params for bucket 18 (45875200 elements, 45875200 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 + module.decoder.layers.5.mlp.experts.linear_fc2.weight7 + module.decoder.layers.5.mlp.experts.linear_fc2.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight6 +Params for bucket 19 (45875200 elements, 45875200 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight0 + module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 +Params for bucket 20 (55050240 elements, 55050240 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight6 + module.decoder.layers.5.mlp.experts.linear_fc1.weight5 +Params for bucket 21 (55050240 elements, 55050240 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight4 + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 +Params for bucket 22 (45875200 elements, 45875200 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight7 +Params for bucket 23 (45875200 elements, 45875200 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 + module.decoder.layers.4.mlp.experts.linear_fc2.weight4 + module.decoder.layers.4.mlp.experts.linear_fc2.weight5 + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 +Params for bucket 24 (55050240 elements, 55050240 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight7 + module.decoder.layers.4.mlp.experts.linear_fc1.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 +Params for bucket 25 (55050240 elements, 55050240 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight5 + module.decoder.layers.4.mlp.experts.linear_fc1.weight4 + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 +Params for bucket 26 (55050240 elements, 55050240 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 +Params for bucket 27 (45875200 elements, 45875200 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight4 + module.decoder.layers.3.mlp.experts.linear_fc2.weight6 + module.decoder.layers.3.mlp.experts.linear_fc2.weight7 + module.decoder.layers.3.mlp.experts.linear_fc2.weight5 + module.decoder.layers.3.mlp.experts.linear_fc2.weight3 +Params for bucket 28 (45875200 elements, 45875200 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 + module.decoder.layers.3.mlp.experts.linear_fc1.weight7 +Params for bucket 29 (55050240 elements, 55050240 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight6 + module.decoder.layers.3.mlp.experts.linear_fc1.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight5 +Params for bucket 30 (55050240 elements, 55050240 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 + module.decoder.layers.3.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc1.weight2 +Params for bucket 31 (45875200 elements, 45875200 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 + module.decoder.layers.2.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc2.weight6 + module.decoder.layers.2.mlp.experts.linear_fc2.weight7 +Params for bucket 32 (45875200 elements, 45875200 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 + module.decoder.layers.2.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 +Params for bucket 33 (55050240 elements, 55050240 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight7 + module.decoder.layers.2.mlp.experts.linear_fc1.weight5 + module.decoder.layers.2.mlp.experts.linear_fc1.weight6 +Params for bucket 34 (55050240 elements, 55050240 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight3 + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 +Params for bucket 35 (45875200 elements, 45875200 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 +Params for bucket 36 (45875200 elements, 45875200 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight6 + module.decoder.layers.1.mlp.experts.linear_fc2.weight4 +Params for bucket 37 (55050240 elements, 55050240 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 + module.decoder.layers.1.mlp.experts.linear_fc1.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 + module.decoder.layers.1.mlp.experts.linear_fc1.weight6 +Params for bucket 38 (55050240 elements, 55050240 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight5 + module.decoder.layers.1.mlp.experts.linear_fc1.weight4 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 +Params for bucket 39 (55050240 elements, 55050240 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 +Params for bucket 40 (45875200 elements, 45875200 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight7 + module.decoder.layers.0.mlp.experts.linear_fc2.weight3 + module.decoder.layers.0.mlp.experts.linear_fc2.weight5 + module.decoder.layers.0.mlp.experts.linear_fc2.weight4 + module.decoder.layers.0.mlp.experts.linear_fc2.weight6 +Params for bucket 41 (45875200 elements, 45875200 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight2 + module.decoder.layers.0.mlp.experts.linear_fc2.weight1 + module.decoder.layers.0.mlp.experts.linear_fc1.weight7 + module.decoder.layers.0.mlp.experts.linear_fc2.weight0 +Params for bucket 42 (55050240 elements, 55050240 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight6 + module.decoder.layers.0.mlp.experts.linear_fc1.weight4 + module.decoder.layers.0.mlp.experts.linear_fc1.weight5 +Params for bucket 43 (55050240 elements, 55050240 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight3 + module.decoder.layers.0.mlp.experts.linear_fc1.weight2 + module.decoder.layers.0.mlp.experts.linear_fc1.weight1 +Params for bucket 44 (18350080 elements, 18350080 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight0 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 15:52:45 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 96 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset: Load the document index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-document_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-sample_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-shuffle_index.npy +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 29073 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-08-26 15:52:45 +done with setup ... +training ... +Overwriting rerun_state_machine.current_iteration from -1 to 0... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (3288.43, 3292.79) + train/valid/test-data-iterators-setup ..........: (68.54, 85.51) +[before the start of training step] datetime: 2026-08-26 15:52:45 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.rerun_state_machine:Result validation enabled +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=0 calls=1 budget=40960 expected_recv=32949 expected_overflow=0 exact_recv_per_rank=[32949, 32753, 32654, 32951, 32512, 32683, 32997, 32645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=0 calls=1 budget=40960 expected_recv=32949 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=0 calls=1 budget=40960 expected_recv=32683 expected_overflow=0 exact_recv_per_rank=[32949, 32753, 32654, 32951, 32512, 32683, 32997, 32645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=0 calls=1 budget=40960 expected_recv=32645 expected_overflow=0 exact_recv_per_rank=[32949, 32753, 32654, 32951, 32512, 32683, 32997, 32645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=1 calls=1 budget=40960 expected_recv=32762 expected_overflow=0 exact_recv_per_rank=[32762, 32841, 32861, 33009, 32485, 32746, 32665, 32775] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=0 calls=1 budget=40960 expected_recv=32683 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=1 calls=1 budget=40960 expected_recv=32762 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=0 calls=1 budget=40960 expected_recv=32645 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=0 calls=1 budget=40960 expected_recv=32951 expected_overflow=0 exact_recv_per_rank=[32949, 32753, 32654, 32951, 32512, 32683, 32997, 32645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=0 calls=1 budget=40960 expected_recv=32753 expected_overflow=0 exact_recv_per_rank=[32949, 32753, 32654, 32951, 32512, 32683, 32997, 32645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=0 calls=1 budget=40960 expected_recv=32512 expected_overflow=0 exact_recv_per_rank=[32949, 32753, 32654, 32951, 32512, 32683, 32997, 32645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32596, 32727, 32724, 32533, 32963, 32718, 32787] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=0 calls=1 budget=40960 expected_recv=32951 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=1 calls=1 budget=40960 expected_recv=32746 expected_overflow=0 exact_recv_per_rank=[32762, 32841, 32861, 33009, 32485, 32746, 32665, 32775] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=1 calls=1 budget=40960 expected_recv=32746 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=0 calls=1 budget=40960 expected_recv=32512 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=0 calls=1 budget=40960 expected_recv=32753 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=1 calls=1 budget=40960 expected_recv=32775 expected_overflow=0 exact_recv_per_rank=[32762, 32841, 32861, 33009, 32485, 32746, 32665, 32775] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=3 calls=1 budget=40960 expected_recv=32788 expected_overflow=0 exact_recv_per_rank=[32788, 32807, 32709, 33096, 32700, 32731, 32691, 32622] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=1 calls=1 budget=40960 expected_recv=32775 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=2 calls=1 budget=40960 expected_recv=32963 expected_overflow=0 exact_recv_per_rank=[33096, 32596, 32727, 32724, 32533, 32963, 32718, 32787] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=3 calls=1 budget=40960 expected_recv=32788 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=1 calls=1 budget=40960 expected_recv=33009 expected_overflow=0 exact_recv_per_rank=[32762, 32841, 32861, 33009, 32485, 32746, 32665, 32775] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=2 calls=1 budget=40960 expected_recv=32963 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=1 calls=1 budget=40960 expected_recv=33009 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=1 calls=1 budget=40960 expected_recv=32485 expected_overflow=0 exact_recv_per_rank=[32762, 32841, 32861, 33009, 32485, 32746, 32665, 32775] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=1 calls=1 budget=40960 expected_recv=32841 expected_overflow=0 exact_recv_per_rank=[32762, 32841, 32861, 33009, 32485, 32746, 32665, 32775] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=1 calls=1 budget=40960 expected_recv=32485 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=2 calls=1 budget=40960 expected_recv=32787 expected_overflow=0 exact_recv_per_rank=[33096, 32596, 32727, 32724, 32533, 32963, 32718, 32787] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=4 calls=1 budget=40960 expected_recv=32664 expected_overflow=0 exact_recv_per_rank=[32664, 32868, 32891, 32965, 32683, 32725, 32758, 32590] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=1 calls=1 budget=40960 expected_recv=32841 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=3 calls=1 budget=40960 expected_recv=32731 expected_overflow=0 exact_recv_per_rank=[32788, 32807, 32709, 33096, 32700, 32731, 32691, 32622] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=2 calls=1 budget=40960 expected_recv=32787 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=4 calls=1 budget=40960 expected_recv=32664 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=3 calls=1 budget=40960 expected_recv=32731 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=2 calls=1 budget=40960 expected_recv=32724 expected_overflow=0 exact_recv_per_rank=[33096, 32596, 32727, 32724, 32533, 32963, 32718, 32787] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=2 calls=1 budget=40960 expected_recv=32724 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=2 calls=1 budget=40960 expected_recv=32533 expected_overflow=0 exact_recv_per_rank=[33096, 32596, 32727, 32724, 32533, 32963, 32718, 32787] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=2 calls=1 budget=40960 expected_recv=32533 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=3 calls=1 budget=40960 expected_recv=32622 expected_overflow=0 exact_recv_per_rank=[32788, 32807, 32709, 33096, 32700, 32731, 32691, 32622] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=2 calls=1 budget=40960 expected_recv=32596 expected_overflow=0 exact_recv_per_rank=[33096, 32596, 32727, 32724, 32533, 32963, 32718, 32787] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=5 calls=1 budget=40960 expected_recv=32958 expected_overflow=0 exact_recv_per_rank=[32958, 32765, 32806, 32999, 32681, 32822, 32673, 32440] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=4 calls=1 budget=40960 expected_recv=32725 expected_overflow=0 exact_recv_per_rank=[32664, 32868, 32891, 32965, 32683, 32725, 32758, 32590] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=3 calls=1 budget=40960 expected_recv=32622 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=2 calls=1 budget=40960 expected_recv=32596 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=5 calls=1 budget=40960 expected_recv=32958 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=4 calls=1 budget=40960 expected_recv=32725 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32788, 32807, 32709, 33096, 32700, 32731, 32691, 32622] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=3 calls=1 budget=40960 expected_recv=32700 expected_overflow=0 exact_recv_per_rank=[32788, 32807, 32709, 33096, 32700, 32731, 32691, 32622] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=3 calls=1 budget=40960 expected_recv=32700 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=4 calls=1 budget=40960 expected_recv=32590 expected_overflow=0 exact_recv_per_rank=[32664, 32868, 32891, 32965, 32683, 32725, 32758, 32590] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=6 calls=1 budget=40960 expected_recv=32994 expected_overflow=0 exact_recv_per_rank=[32994, 32777, 32775, 32427, 32642, 32908, 32812, 32809] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=3 calls=1 budget=40960 expected_recv=32807 expected_overflow=0 exact_recv_per_rank=[32788, 32807, 32709, 33096, 32700, 32731, 32691, 32622] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=5 calls=1 budget=40960 expected_recv=32822 expected_overflow=0 exact_recv_per_rank=[32958, 32765, 32806, 32999, 32681, 32822, 32673, 32440] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=4 calls=1 budget=40960 expected_recv=32590 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=6 calls=1 budget=40960 expected_recv=32994 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=3 calls=1 budget=40960 expected_recv=32807 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=5 calls=1 budget=40960 expected_recv=32822 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=4 calls=1 budget=40960 expected_recv=32965 expected_overflow=0 exact_recv_per_rank=[32664, 32868, 32891, 32965, 32683, 32725, 32758, 32590] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=4 calls=1 budget=40960 expected_recv=32683 expected_overflow=0 exact_recv_per_rank=[32664, 32868, 32891, 32965, 32683, 32725, 32758, 32590] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=4 calls=1 budget=40960 expected_recv=32965 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=0 calls=1 budget=40960 expected_recv=32997 expected_overflow=0 exact_recv_per_rank=[32949, 32753, 32654, 32951, 32512, 32683, 32997, 32645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=4 calls=1 budget=40960 expected_recv=32683 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=5 calls=1 budget=40960 expected_recv=32440 expected_overflow=0 exact_recv_per_rank=[32958, 32765, 32806, 32999, 32681, 32822, 32673, 32440] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=7 calls=1 budget=40960 expected_recv=32501 expected_overflow=0 exact_recv_per_rank=[32501, 32704, 32850, 32435, 33096, 32781, 33034, 32743] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=4 calls=1 budget=40960 expected_recv=32868 expected_overflow=0 exact_recv_per_rank=[32664, 32868, 32891, 32965, 32683, 32725, 32758, 32590] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=6 calls=1 budget=40960 expected_recv=32908 expected_overflow=0 exact_recv_per_rank=[32994, 32777, 32775, 32427, 32642, 32908, 32812, 32809] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=5 calls=1 budget=40960 expected_recv=32440 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=7 calls=1 budget=40960 expected_recv=32501 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=4 calls=1 budget=40960 expected_recv=32868 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=6 calls=1 budget=40960 expected_recv=32908 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=0 calls=1 budget=40960 expected_recv=32997 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=0 calls=1 budget=40960 expected_recv=32654 expected_overflow=0 exact_recv_per_rank=[32949, 32753, 32654, 32951, 32512, 32683, 32997, 32645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=5 calls=1 budget=40960 expected_recv=32999 expected_overflow=0 exact_recv_per_rank=[32958, 32765, 32806, 32999, 32681, 32822, 32673, 32440] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=5 calls=1 budget=40960 expected_recv=32681 expected_overflow=0 exact_recv_per_rank=[32958, 32765, 32806, 32999, 32681, 32822, 32673, 32440] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=5 calls=1 budget=40960 expected_recv=32999 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=5 calls=1 budget=40960 expected_recv=32681 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=6 calls=1 budget=40960 expected_recv=32809 expected_overflow=0 exact_recv_per_rank=[32994, 32777, 32775, 32427, 32642, 32908, 32812, 32809] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=8 calls=1 budget=40960 expected_recv=32565 expected_overflow=0 exact_recv_per_rank=[32565, 32402, 32881, 32878, 32730, 32846, 33023, 32819] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=7 calls=1 budget=40960 expected_recv=32781 expected_overflow=0 exact_recv_per_rank=[32501, 32704, 32850, 32435, 33096, 32781, 33034, 32743] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=6 calls=1 budget=40960 expected_recv=32809 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=5 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[32958, 32765, 32806, 32999, 32681, 32822, 32673, 32440] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=0 calls=1 budget=40960 expected_recv=32654 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=8 calls=1 budget=40960 expected_recv=32565 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=7 calls=1 budget=40960 expected_recv=32781 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=5 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=6 calls=1 budget=40960 expected_recv=32427 expected_overflow=0 exact_recv_per_rank=[32994, 32777, 32775, 32427, 32642, 32908, 32812, 32809] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=6 calls=1 budget=40960 expected_recv=32642 expected_overflow=0 exact_recv_per_rank=[32994, 32777, 32775, 32427, 32642, 32908, 32812, 32809] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=1 calls=1 budget=40960 expected_recv=32665 expected_overflow=0 exact_recv_per_rank=[32762, 32841, 32861, 33009, 32485, 32746, 32665, 32775] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=6 calls=1 budget=40960 expected_recv=32427 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=6 calls=1 budget=40960 expected_recv=32642 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=1 calls=1 budget=40960 expected_recv=32665 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=7 calls=1 budget=40960 expected_recv=32743 expected_overflow=0 exact_recv_per_rank=[32501, 32704, 32850, 32435, 33096, 32781, 33034, 32743] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=9 calls=1 budget=40960 expected_recv=32864 expected_overflow=0 exact_recv_per_rank=[32864, 32469, 32789, 32447, 32867, 32835, 33063, 32810] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=8 calls=1 budget=40960 expected_recv=32846 expected_overflow=0 exact_recv_per_rank=[32565, 32402, 32881, 32878, 32730, 32846, 33023, 32819] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=6 calls=1 budget=40960 expected_recv=32777 expected_overflow=0 exact_recv_per_rank=[32994, 32777, 32775, 32427, 32642, 32908, 32812, 32809] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=7 calls=1 budget=40960 expected_recv=32743 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=9 calls=1 budget=40960 expected_recv=32864 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=8 calls=1 budget=40960 expected_recv=32846 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=6 calls=1 budget=40960 expected_recv=32777 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=1 calls=1 budget=40960 expected_recv=32861 expected_overflow=0 exact_recv_per_rank=[32762, 32841, 32861, 33009, 32485, 32746, 32665, 32775] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32501, 32704, 32850, 32435, 33096, 32781, 33034, 32743] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=7 calls=1 budget=40960 expected_recv=32435 expected_overflow=0 exact_recv_per_rank=[32501, 32704, 32850, 32435, 33096, 32781, 33034, 32743] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=1 calls=1 budget=40960 expected_recv=32861 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=7 calls=1 budget=40960 expected_recv=32435 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=2 calls=1 budget=40960 expected_recv=32718 expected_overflow=0 exact_recv_per_rank=[33096, 32596, 32727, 32724, 32533, 32963, 32718, 32787] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=2 calls=1 budget=40960 expected_recv=32718 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=8 calls=1 budget=40960 expected_recv=32819 expected_overflow=0 exact_recv_per_rank=[32565, 32402, 32881, 32878, 32730, 32846, 33023, 32819] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=9 calls=1 budget=40960 expected_recv=32835 expected_overflow=0 exact_recv_per_rank=[32864, 32469, 32789, 32447, 32867, 32835, 33063, 32810] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=7 calls=1 budget=40960 expected_recv=32704 expected_overflow=0 exact_recv_per_rank=[32501, 32704, 32850, 32435, 33096, 32781, 33034, 32743] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=8 calls=1 budget=40960 expected_recv=32819 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=9 calls=1 budget=40960 expected_recv=32835 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=7 calls=1 budget=40960 expected_recv=32704 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=8 calls=1 budget=40960 expected_recv=32730 expected_overflow=0 exact_recv_per_rank=[32565, 32402, 32881, 32878, 32730, 32846, 33023, 32819] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=2 calls=1 budget=40960 expected_recv=32727 expected_overflow=0 exact_recv_per_rank=[33096, 32596, 32727, 32724, 32533, 32963, 32718, 32787] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=1 tokens_per_ep_rank=[32949, 32753, 32654, 32951, 32512, 32683, 32997, 32645] avg=32768.0000 max=32997 max_rank=6 min=32512 min_rank=4 imbalance_ratio=1.006989 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=8 calls=1 budget=40960 expected_recv=32878 expected_overflow=0 exact_recv_per_rank=[32565, 32402, 32881, 32878, 32730, 32846, 33023, 32819] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=8 calls=1 budget=40960 expected_recv=32730 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=2 calls=1 budget=40960 expected_recv=32727 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=3 calls=1 budget=40960 expected_recv=32691 expected_overflow=0 exact_recv_per_rank=[32788, 32807, 32709, 33096, 32700, 32731, 32691, 32622] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=8 calls=1 budget=40960 expected_recv=32878 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=3 calls=1 budget=40960 expected_recv=32691 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=9 calls=1 budget=40960 expected_recv=32810 expected_overflow=0 exact_recv_per_rank=[32864, 32469, 32789, 32447, 32867, 32835, 33063, 32810] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=1 tokens_per_ep_rank=[32949, 32753, 32654, 32951, 32512, 32683, 32997, 32645] avg=32768.0000 max=32997 max_rank=6 min=32512 min_rank=4 imbalance_ratio=1.006989 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=9 calls=1 budget=40960 expected_recv=32810 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=8 calls=1 budget=40960 expected_recv=32402 expected_overflow=0 exact_recv_per_rank=[32565, 32402, 32881, 32878, 32730, 32846, 33023, 32819] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=1 tokens_per_ep_rank=[32949, 32753, 32654, 32951, 32512, 32683, 32997, 32645] avg=32768.0000 max=32997 max_rank=6 min=32512 min_rank=4 imbalance_ratio=1.006989 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=8 calls=1 budget=40960 expected_recv=32402 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=9 calls=1 budget=40960 expected_recv=32867 expected_overflow=0 exact_recv_per_rank=[32864, 32469, 32789, 32447, 32867, 32835, 33063, 32810] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=3 calls=1 budget=40960 expected_recv=32709 expected_overflow=0 exact_recv_per_rank=[32788, 32807, 32709, 33096, 32700, 32731, 32691, 32622] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=9 calls=1 budget=40960 expected_recv=32867 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=1 tokens_per_ep_rank=[32949, 32753, 32654, 32951, 32512, 32683, 32997, 32645] avg=32768.0000 max=32997 max_rank=6 min=32512 min_rank=4 imbalance_ratio=1.006989 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=3 calls=1 budget=40960 expected_recv=32709 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=9 calls=1 budget=40960 expected_recv=32447 expected_overflow=0 exact_recv_per_rank=[32864, 32469, 32789, 32447, 32867, 32835, 33063, 32810] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=2 tokens_per_ep_rank=[32762, 32841, 32861, 33009, 32485, 32746, 32665, 32775] avg=32768.0000 max=33009 max_rank=3 min=32485 min_rank=4 imbalance_ratio=1.007355 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=4 calls=1 budget=40960 expected_recv=32758 expected_overflow=0 exact_recv_per_rank=[32664, 32868, 32891, 32965, 32683, 32725, 32758, 32590] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=9 calls=1 budget=40960 expected_recv=32447 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=4 calls=1 budget=40960 expected_recv=32758 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=9 calls=1 budget=40960 expected_recv=32469 expected_overflow=0 exact_recv_per_rank=[32864, 32469, 32789, 32447, 32867, 32835, 33063, 32810] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=1 tokens_per_ep_rank=[32949, 32753, 32654, 32951, 32512, 32683, 32997, 32645] avg=32768.0000 max=32997 max_rank=6 min=32512 min_rank=4 imbalance_ratio=1.006989 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=9 calls=1 budget=40960 expected_recv=32469 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=2 tokens_per_ep_rank=[32762, 32841, 32861, 33009, 32485, 32746, 32665, 32775] avg=32768.0000 max=33009 max_rank=3 min=32485 min_rank=4 imbalance_ratio=1.007355 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=2 tokens_per_ep_rank=[32762, 32841, 32861, 33009, 32485, 32746, 32665, 32775] avg=32768.0000 max=33009 max_rank=3 min=32485 min_rank=4 imbalance_ratio=1.007355 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=4 calls=1 budget=40960 expected_recv=32891 expected_overflow=0 exact_recv_per_rank=[32664, 32868, 32891, 32965, 32683, 32725, 32758, 32590] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=1 tokens_per_ep_rank=[32949, 32753, 32654, 32951, 32512, 32683, 32997, 32645] avg=32768.0000 max=32997 max_rank=6 min=32512 min_rank=4 imbalance_ratio=1.006989 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=4 calls=1 budget=40960 expected_recv=32891 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=1 tokens_per_ep_rank=[32949, 32753, 32654, 32951, 32512, 32683, 32997, 32645] avg=32768.0000 max=32997 max_rank=6 min=32512 min_rank=4 imbalance_ratio=1.006989 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=2 tokens_per_ep_rank=[32762, 32841, 32861, 33009, 32485, 32746, 32665, 32775] avg=32768.0000 max=33009 max_rank=3 min=32485 min_rank=4 imbalance_ratio=1.007355 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=5 calls=1 budget=40960 expected_recv=32673 expected_overflow=0 exact_recv_per_rank=[32958, 32765, 32806, 32999, 32681, 32822, 32673, 32440] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=3 tokens_per_ep_rank=[33098, 32596, 32727, 32724, 32531, 32963, 32718, 32787] avg=32768.0000 max=33098 max_rank=0 min=32531 min_rank=4 imbalance_ratio=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=5 calls=1 budget=40960 expected_recv=32673 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=1 tokens_per_ep_rank=[32949, 32753, 32654, 32951, 32512, 32683, 32997, 32645] avg=32768.0000 max=32997 max_rank=6 min=32512 min_rank=4 imbalance_ratio=1.006989 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=1 tokens_per_ep_rank=[32949, 32753, 32654, 32951, 32512, 32683, 32997, 32645] avg=32768.0000 max=32997 max_rank=6 min=32512 min_rank=4 imbalance_ratio=1.006989 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=1 tokens_per_ep_rank=[32949, 32753, 32654, 32951, 32512, 32683, 32997, 32645] avg=32768.0000 max=32997 max_rank=6 min=32512 min_rank=4 imbalance_ratio=1.006989 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=2 tokens_per_ep_rank=[32762, 32841, 32861, 33009, 32485, 32746, 32665, 32775] avg=32768.0000 max=33009 max_rank=3 min=32485 min_rank=4 imbalance_ratio=1.007355 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=3 tokens_per_ep_rank=[33096, 32596, 32727, 32724, 32533, 32963, 32718, 32787] avg=32768.0000 max=33096 max_rank=0 min=32533 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=2 max_load_reduction_pct=0.000060 moved_to_replicas=2 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=3 tokens_per_ep_rank=[33098, 32596, 32727, 32724, 32531, 32963, 32718, 32787] avg=32768.0000 max=33098 max_rank=0 min=32531 min_rank=4 imbalance_ratio=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=5 calls=1 budget=40960 expected_recv=32806 expected_overflow=0 exact_recv_per_rank=[32958, 32765, 32806, 32999, 32681, 32822, 32673, 32440] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=1 tokens_per_ep_rank=[32949, 32753, 32654, 32951, 32512, 32683, 32997, 32645] avg=32768.0000 max=32997 max_rank=6 min=32512 min_rank=4 imbalance_ratio=1.006989 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=5 calls=1 budget=40960 expected_recv=32806 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=1 tokens_per_ep_rank=[32949, 32753, 32654, 32951, 32512, 32683, 32997, 32645] avg=32768.0000 max=32997 max_rank=6 min=32512 min_rank=4 imbalance_ratio=1.006989 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=2 tokens_per_ep_rank=[32762, 32841, 32861, 33009, 32485, 32746, 32665, 32775] avg=32768.0000 max=33009 max_rank=3 min=32485 min_rank=4 imbalance_ratio=1.007355 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=6 calls=1 budget=40960 expected_recv=32812 expected_overflow=0 exact_recv_per_rank=[32994, 32777, 32775, 32427, 32642, 32908, 32812, 32809] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=3 tokens_per_ep_rank=[33096, 32596, 32727, 32724, 32533, 32963, 32718, 32787] avg=32768.0000 max=33096 max_rank=0 min=32533 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=2 max_load_reduction_pct=0.000060 moved_to_replicas=2 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=2 tokens_per_ep_rank=[32762, 32841, 32861, 33009, 32485, 32746, 32665, 32775] avg=32768.0000 max=33009 max_rank=3 min=32485 min_rank=4 imbalance_ratio=1.007355 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=6 calls=1 budget=40960 expected_recv=32812 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=4 tokens_per_ep_rank=[32788, 32807, 32709, 33198, 32598, 32731, 32691, 32622] avg=32768.0000 max=33198 max_rank=3 min=32598 min_rank=4 imbalance_ratio=1.013123 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=2 tokens_per_ep_rank=[32762, 32841, 32861, 33009, 32485, 32746, 32665, 32775] avg=32768.0000 max=33009 max_rank=3 min=32485 min_rank=4 imbalance_ratio=1.007355 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=2 tokens_per_ep_rank=[32762, 32841, 32861, 33009, 32485, 32746, 32665, 32775] avg=32768.0000 max=33009 max_rank=3 min=32485 min_rank=4 imbalance_ratio=1.007355 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=3 tokens_per_ep_rank=[33098, 32596, 32727, 32724, 32531, 32963, 32718, 32787] avg=32768.0000 max=33098 max_rank=0 min=32531 min_rank=4 imbalance_ratio=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=2 tokens_per_ep_rank=[32762, 32841, 32861, 33009, 32485, 32746, 32665, 32775] avg=32768.0000 max=33009 max_rank=3 min=32485 min_rank=4 imbalance_ratio=1.007355 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=6 calls=1 budget=40960 expected_recv=32775 expected_overflow=0 exact_recv_per_rank=[32994, 32777, 32775, 32427, 32642, 32908, 32812, 32809] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=4 tokens_per_ep_rank=[32788, 32807, 32709, 33198, 32598, 32731, 32691, 32622] avg=32768.0000 max=33198 max_rank=3 min=32598 min_rank=4 imbalance_ratio=1.013123 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=4 tokens_per_ep_rank=[32788, 32807, 32709, 33096, 32700, 32731, 32691, 32622] avg=32768.0000 max=33096 max_rank=3 min=32622 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=102 max_load_reduction_pct=0.003072 moved_to_replicas=102 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=6 calls=1 budget=40960 expected_recv=32775 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=2 tokens_per_ep_rank=[32762, 32841, 32861, 33009, 32485, 32746, 32665, 32775] avg=32768.0000 max=33009 max_rank=3 min=32485 min_rank=4 imbalance_ratio=1.007355 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=7 calls=1 budget=40960 expected_recv=33034 expected_overflow=0 exact_recv_per_rank=[32501, 32704, 32850, 32435, 33096, 32781, 33034, 32743] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=3 tokens_per_ep_rank=[33096, 32596, 32727, 32724, 32533, 32963, 32718, 32787] avg=32768.0000 max=33096 max_rank=0 min=32533 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=2 max_load_reduction_pct=0.000060 moved_to_replicas=2 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=7 calls=1 budget=40960 expected_recv=33034 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=2 tokens_per_ep_rank=[32762, 32841, 32861, 33009, 32485, 32746, 32665, 32775] avg=32768.0000 max=33009 max_rank=3 min=32485 min_rank=4 imbalance_ratio=1.007355 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=4 tokens_per_ep_rank=[32788, 32807, 32709, 33096, 32700, 32731, 32691, 32622] avg=32768.0000 max=33096 max_rank=3 min=32622 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=102 max_load_reduction_pct=0.003072 moved_to_replicas=102 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=3 tokens_per_ep_rank=[33098, 32596, 32727, 32724, 32531, 32963, 32718, 32787] avg=32768.0000 max=33098 max_rank=0 min=32531 min_rank=4 imbalance_ratio=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=5 tokens_per_ep_rank=[32664, 32868, 32891, 32965, 32683, 32725, 32758, 32590] avg=32768.0000 max=32965 max_rank=3 min=32590 min_rank=7 imbalance_ratio=1.006012 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=3 tokens_per_ep_rank=[33098, 32596, 32727, 32724, 32531, 32963, 32718, 32787] avg=32768.0000 max=33098 max_rank=0 min=32531 min_rank=4 imbalance_ratio=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=7 calls=1 budget=40960 expected_recv=32850 expected_overflow=0 exact_recv_per_rank=[32501, 32704, 32850, 32435, 33096, 32781, 33034, 32743] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=4 tokens_per_ep_rank=[32788, 32807, 32709, 33198, 32598, 32731, 32691, 32622] avg=32768.0000 max=33198 max_rank=3 min=32598 min_rank=4 imbalance_ratio=1.013123 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=3 tokens_per_ep_rank=[33096, 32596, 32727, 32724, 32533, 32963, 32718, 32787] avg=32768.0000 max=33096 max_rank=0 min=32533 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=2 max_load_reduction_pct=0.000060 moved_to_replicas=2 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=7 calls=1 budget=40960 expected_recv=32850 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=5 tokens_per_ep_rank=[32664, 32868, 32891, 32965, 32683, 32725, 32758, 32590] avg=32768.0000 max=32965 max_rank=3 min=32590 min_rank=7 imbalance_ratio=1.006012 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=5 tokens_per_ep_rank=[32664, 32868, 32891, 32965, 32683, 32725, 32758, 32590] avg=32768.0000 max=32965 max_rank=3 min=32590 min_rank=7 imbalance_ratio=1.006012 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=3 tokens_per_ep_rank=[33098, 32596, 32727, 32724, 32531, 32963, 32718, 32787] avg=32768.0000 max=33098 max_rank=0 min=32531 min_rank=4 imbalance_ratio=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=8 calls=1 budget=40960 expected_recv=33023 expected_overflow=0 exact_recv_per_rank=[32565, 32402, 32881, 32878, 32730, 32846, 33023, 32819] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=3 tokens_per_ep_rank=[33096, 32596, 32727, 32724, 32533, 32963, 32718, 32787] avg=32768.0000 max=33096 max_rank=0 min=32533 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=2 max_load_reduction_pct=0.000060 moved_to_replicas=2 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=8 calls=1 budget=40960 expected_recv=33023 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=4 tokens_per_ep_rank=[32788, 32807, 32709, 33096, 32700, 32731, 32691, 32622] avg=32768.0000 max=33096 max_rank=3 min=32622 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=102 max_load_reduction_pct=0.003072 moved_to_replicas=102 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=5 tokens_per_ep_rank=[32664, 32868, 32891, 32965, 32683, 32725, 32758, 32590] avg=32768.0000 max=32965 max_rank=3 min=32590 min_rank=7 imbalance_ratio=1.006012 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=3 tokens_per_ep_rank=[33096, 32596, 32727, 32724, 32533, 32963, 32718, 32787] avg=32768.0000 max=33096 max_rank=0 min=32533 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=2 max_load_reduction_pct=0.000060 moved_to_replicas=2 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=4 tokens_per_ep_rank=[32788, 32807, 32709, 33198, 32598, 32731, 32691, 32622] avg=32768.0000 max=33198 max_rank=3 min=32598 min_rank=4 imbalance_ratio=1.013123 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=6 tokens_per_ep_rank=[32958, 32765, 32806, 32999, 32681, 32822, 32673, 32440] avg=32768.0000 max=32999 max_rank=3 min=32440 min_rank=7 imbalance_ratio=1.007050 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=8 calls=1 budget=40960 expected_recv=32881 expected_overflow=0 exact_recv_per_rank=[32565, 32402, 32881, 32878, 32730, 32846, 33023, 32819] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=4 tokens_per_ep_rank=[32788, 32807, 32709, 33198, 32598, 32731, 32691, 32622] avg=32768.0000 max=33198 max_rank=3 min=32598 min_rank=4 imbalance_ratio=1.013123 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=8 calls=1 budget=40960 expected_recv=32881 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=6 tokens_per_ep_rank=[32958, 32765, 32806, 32999, 32681, 32822, 32673, 32440] avg=32768.0000 max=32999 max_rank=3 min=32440 min_rank=7 imbalance_ratio=1.007050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=4 tokens_per_ep_rank=[32788, 32807, 32709, 33096, 32700, 32731, 32691, 32622] avg=32768.0000 max=33096 max_rank=3 min=32622 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=102 max_load_reduction_pct=0.003072 moved_to_replicas=102 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=5 tokens_per_ep_rank=[32664, 32868, 32891, 32965, 32683, 32725, 32758, 32590] avg=32768.0000 max=32965 max_rank=3 min=32590 min_rank=7 imbalance_ratio=1.006012 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=9 calls=1 budget=40960 expected_recv=33063 expected_overflow=0 exact_recv_per_rank=[32864, 32469, 32789, 32447, 32867, 32835, 33063, 32810] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=6 tokens_per_ep_rank=[32958, 32765, 32806, 32999, 32681, 32822, 32673, 32440] avg=32768.0000 max=32999 max_rank=3 min=32440 min_rank=7 imbalance_ratio=1.007050 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=4 tokens_per_ep_rank=[32788, 32807, 32709, 33198, 32598, 32731, 32691, 32622] avg=32768.0000 max=33198 max_rank=3 min=32598 min_rank=4 imbalance_ratio=1.013123 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=4 tokens_per_ep_rank=[32788, 32807, 32709, 33096, 32700, 32731, 32691, 32622] avg=32768.0000 max=33096 max_rank=3 min=32622 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=102 max_load_reduction_pct=0.003072 moved_to_replicas=102 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=9 calls=1 budget=40960 expected_recv=33063 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=6 tokens_per_ep_rank=[32958, 32765, 32806, 32999, 32681, 32822, 32673, 32440] avg=32768.0000 max=32999 max_rank=3 min=32440 min_rank=7 imbalance_ratio=1.007050 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=5 tokens_per_ep_rank=[32664, 32868, 32891, 32965, 32683, 32725, 32758, 32590] avg=32768.0000 max=32965 max_rank=3 min=32590 min_rank=7 imbalance_ratio=1.006012 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=5 tokens_per_ep_rank=[32664, 32868, 32891, 32965, 32683, 32725, 32758, 32590] avg=32768.0000 max=32965 max_rank=3 min=32590 min_rank=7 imbalance_ratio=1.006012 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=9 calls=1 budget=40960 expected_recv=32789 expected_overflow=0 exact_recv_per_rank=[32864, 32469, 32789, 32447, 32867, 32835, 33063, 32810] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=4 tokens_per_ep_rank=[32788, 32807, 32709, 33096, 32700, 32731, 32691, 32622] avg=32768.0000 max=33096 max_rank=3 min=32622 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=102 max_load_reduction_pct=0.003072 moved_to_replicas=102 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=7 tokens_per_ep_rank=[32994, 32777, 32775, 32427, 32642, 32908, 32812, 32809] avg=32768.0000 max=32994 max_rank=0 min=32427 min_rank=3 imbalance_ratio=1.006897 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=9 calls=1 budget=40960 expected_recv=32789 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=7 tokens_per_ep_rank=[32994, 32777, 32775, 32427, 32642, 32908, 32812, 32809] avg=32768.0000 max=32994 max_rank=0 min=32427 min_rank=3 imbalance_ratio=1.006897 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=5 tokens_per_ep_rank=[32664, 32868, 32891, 32965, 32683, 32725, 32758, 32590] avg=32768.0000 max=32965 max_rank=3 min=32590 min_rank=7 imbalance_ratio=1.006012 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=6 tokens_per_ep_rank=[32958, 32765, 32806, 32999, 32681, 32822, 32673, 32440] avg=32768.0000 max=32999 max_rank=3 min=32440 min_rank=7 imbalance_ratio=1.007050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=5 tokens_per_ep_rank=[32664, 32868, 32891, 32965, 32683, 32725, 32758, 32590] avg=32768.0000 max=32965 max_rank=3 min=32590 min_rank=7 imbalance_ratio=1.006012 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=7 tokens_per_ep_rank=[32994, 32777, 32775, 32427, 32642, 32908, 32812, 32809] avg=32768.0000 max=32994 max_rank=0 min=32427 min_rank=3 imbalance_ratio=1.006897 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=1 tokens_per_ep_rank=[32949, 32753, 32654, 32951, 32512, 32683, 32997, 32645] avg=32768.0000 max=32997 max_rank=6 min=32512 min_rank=4 imbalance_ratio=1.006989 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=5 tokens_per_ep_rank=[32664, 32868, 32891, 32965, 32683, 32725, 32758, 32590] avg=32768.0000 max=32965 max_rank=3 min=32590 min_rank=7 imbalance_ratio=1.006012 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=7 tokens_per_ep_rank=[32994, 32777, 32775, 32427, 32642, 32908, 32812, 32809] avg=32768.0000 max=32994 max_rank=0 min=32427 min_rank=3 imbalance_ratio=1.006897 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=5 tokens_per_ep_rank=[32664, 32868, 32891, 32965, 32683, 32725, 32758, 32590] avg=32768.0000 max=32965 max_rank=3 min=32590 min_rank=7 imbalance_ratio=1.006012 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=6 tokens_per_ep_rank=[32958, 32765, 32806, 32999, 32681, 32822, 32673, 32440] avg=32768.0000 max=32999 max_rank=3 min=32440 min_rank=7 imbalance_ratio=1.007050 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=6 tokens_per_ep_rank=[32958, 32765, 32806, 32999, 32681, 32822, 32673, 32440] avg=32768.0000 max=32999 max_rank=3 min=32440 min_rank=7 imbalance_ratio=1.007050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=8 tokens_per_ep_rank=[32501, 32704, 32850, 32380, 33151, 32781, 33034, 32743] avg=32768.0000 max=33151 max_rank=4 min=32380 min_rank=3 imbalance_ratio=1.011688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=5 tokens_per_ep_rank=[32664, 32868, 32891, 32965, 32683, 32725, 32758, 32590] avg=32768.0000 max=32965 max_rank=3 min=32590 min_rank=7 imbalance_ratio=1.006012 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=1 tokens_per_ep_rank=[32949, 32753, 32654, 32951, 32512, 32683, 32997, 32645] avg=32768.0000 max=32997 max_rank=6 min=32512 min_rank=4 imbalance_ratio=1.006989 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=8 tokens_per_ep_rank=[32501, 32704, 32850, 32380, 33151, 32781, 33034, 32743] avg=32768.0000 max=33151 max_rank=4 min=32380 min_rank=3 imbalance_ratio=1.011688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=1 tokens_per_ep_rank=[32949, 32753, 32654, 32951, 32512, 32683, 32997, 32645] avg=32768.0000 max=32997 max_rank=6 min=32512 min_rank=4 imbalance_ratio=1.006989 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=6 tokens_per_ep_rank=[32958, 32765, 32806, 32999, 32681, 32822, 32673, 32440] avg=32768.0000 max=32999 max_rank=3 min=32440 min_rank=7 imbalance_ratio=1.007050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=6 tokens_per_ep_rank=[32958, 32765, 32806, 32999, 32681, 32822, 32673, 32440] avg=32768.0000 max=32999 max_rank=3 min=32440 min_rank=7 imbalance_ratio=1.007050 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=7 tokens_per_ep_rank=[32994, 32777, 32775, 32427, 32642, 32908, 32812, 32809] avg=32768.0000 max=32994 max_rank=0 min=32427 min_rank=3 imbalance_ratio=1.006897 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=8 tokens_per_ep_rank=[32501, 32704, 32850, 32435, 33096, 32781, 33034, 32743] avg=32768.0000 max=33096 max_rank=4 min=32435 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=55 max_load_reduction_pct=0.001659 moved_to_replicas=55 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=8 tokens_per_ep_rank=[32501, 32704, 32850, 32435, 33096, 32781, 33034, 32743] avg=32768.0000 max=33096 max_rank=4 min=32435 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=55 max_load_reduction_pct=0.001659 moved_to_replicas=55 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=6 tokens_per_ep_rank=[32958, 32765, 32806, 32999, 32681, 32822, 32673, 32440] avg=32768.0000 max=32999 max_rank=3 min=32440 min_rank=7 imbalance_ratio=1.007050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=6 tokens_per_ep_rank=[32958, 32765, 32806, 32999, 32681, 32822, 32673, 32440] avg=32768.0000 max=32999 max_rank=3 min=32440 min_rank=7 imbalance_ratio=1.007050 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=2 tokens_per_ep_rank=[32762, 32841, 32861, 33009, 32485, 32746, 32665, 32775] avg=32768.0000 max=33009 max_rank=3 min=32485 min_rank=4 imbalance_ratio=1.007355 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=7 tokens_per_ep_rank=[32994, 32777, 32775, 32427, 32642, 32908, 32812, 32809] avg=32768.0000 max=32994 max_rank=0 min=32427 min_rank=3 imbalance_ratio=1.006897 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=1 tokens_per_ep_rank=[32949, 32753, 32654, 32951, 32512, 32683, 32997, 32645] avg=32768.0000 max=32997 max_rank=6 min=32512 min_rank=4 imbalance_ratio=1.006989 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=7 tokens_per_ep_rank=[32994, 32777, 32775, 32427, 32642, 32908, 32812, 32809] avg=32768.0000 max=32994 max_rank=0 min=32427 min_rank=3 imbalance_ratio=1.006897 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=9 tokens_per_ep_rank=[32565, 32402, 32881, 32878, 32730, 32846, 33023, 32819] avg=32768.0000 max=33023 max_rank=6 min=32402 min_rank=1 imbalance_ratio=1.007782 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=6 tokens_per_ep_rank=[32958, 32765, 32806, 32999, 32681, 32822, 32673, 32440] avg=32768.0000 max=32999 max_rank=3 min=32440 min_rank=7 imbalance_ratio=1.007050 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=2 tokens_per_ep_rank=[32762, 32841, 32861, 33009, 32485, 32746, 32665, 32775] avg=32768.0000 max=33009 max_rank=3 min=32485 min_rank=4 imbalance_ratio=1.007355 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=9 tokens_per_ep_rank=[32565, 32402, 32881, 32878, 32730, 32846, 33023, 32819] avg=32768.0000 max=33023 max_rank=6 min=32402 min_rank=1 imbalance_ratio=1.007782 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=7 tokens_per_ep_rank=[32994, 32777, 32775, 32427, 32642, 32908, 32812, 32809] avg=32768.0000 max=32994 max_rank=0 min=32427 min_rank=3 imbalance_ratio=1.006897 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=8 tokens_per_ep_rank=[32501, 32704, 32850, 32380, 33151, 32781, 33034, 32743] avg=32768.0000 max=33151 max_rank=4 min=32380 min_rank=3 imbalance_ratio=1.011688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=7 tokens_per_ep_rank=[32994, 32777, 32775, 32427, 32642, 32908, 32812, 32809] avg=32768.0000 max=32994 max_rank=0 min=32427 min_rank=3 imbalance_ratio=1.006897 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=2 tokens_per_ep_rank=[32762, 32841, 32861, 33009, 32485, 32746, 32665, 32775] avg=32768.0000 max=33009 max_rank=3 min=32485 min_rank=4 imbalance_ratio=1.007355 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=9 tokens_per_ep_rank=[32565, 32402, 32881, 32878, 32730, 32846, 33023, 32819] avg=32768.0000 max=33023 max_rank=6 min=32402 min_rank=1 imbalance_ratio=1.007782 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=9 tokens_per_ep_rank=[32565, 32402, 32881, 32878, 32730, 32846, 33023, 32819] avg=32768.0000 max=33023 max_rank=6 min=32402 min_rank=1 imbalance_ratio=1.007782 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=7 tokens_per_ep_rank=[32994, 32777, 32775, 32427, 32642, 32908, 32812, 32809] avg=32768.0000 max=32994 max_rank=0 min=32427 min_rank=3 imbalance_ratio=1.006897 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=8 tokens_per_ep_rank=[32501, 32704, 32850, 32435, 33096, 32781, 33034, 32743] avg=32768.0000 max=33096 max_rank=4 min=32435 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=55 max_load_reduction_pct=0.001659 moved_to_replicas=55 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=7 tokens_per_ep_rank=[32994, 32777, 32775, 32427, 32642, 32908, 32812, 32809] avg=32768.0000 max=32994 max_rank=0 min=32427 min_rank=3 imbalance_ratio=1.006897 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=3 tokens_per_ep_rank=[33098, 32596, 32727, 32724, 32531, 32963, 32718, 32787] avg=32768.0000 max=33098 max_rank=0 min=32531 min_rank=4 imbalance_ratio=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=2 tokens_per_ep_rank=[32762, 32841, 32861, 33009, 32485, 32746, 32665, 32775] avg=32768.0000 max=33009 max_rank=3 min=32485 min_rank=4 imbalance_ratio=1.007355 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=8 tokens_per_ep_rank=[32501, 32704, 32850, 32380, 33151, 32781, 33034, 32743] avg=32768.0000 max=33151 max_rank=4 min=32380 min_rank=3 imbalance_ratio=1.011688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=7 tokens_per_ep_rank=[32994, 32777, 32775, 32427, 32642, 32908, 32812, 32809] avg=32768.0000 max=32994 max_rank=0 min=32427 min_rank=3 imbalance_ratio=1.006897 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=10 tokens_per_ep_rank=[32864, 32469, 32789, 32447, 32867, 32835, 33063, 32810] avg=32768.0000 max=33063 max_rank=6 min=32447 min_rank=3 imbalance_ratio=1.009003 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=10 tokens_per_ep_rank=[32864, 32469, 32789, 32447, 32867, 32835, 33063, 32810] avg=32768.0000 max=33063 max_rank=6 min=32447 min_rank=3 imbalance_ratio=1.009003 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=3 tokens_per_ep_rank=[33096, 32596, 32727, 32724, 32533, 32963, 32718, 32787] avg=32768.0000 max=33096 max_rank=0 min=32533 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=2 max_load_reduction_pct=0.000060 moved_to_replicas=2 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=8 tokens_per_ep_rank=[32501, 32704, 32850, 32435, 33096, 32781, 33034, 32743] avg=32768.0000 max=33096 max_rank=4 min=32435 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=55 max_load_reduction_pct=0.001659 moved_to_replicas=55 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=9 tokens_per_ep_rank=[32565, 32402, 32881, 32878, 32730, 32846, 33023, 32819] avg=32768.0000 max=33023 max_rank=6 min=32402 min_rank=1 imbalance_ratio=1.007782 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=10 tokens_per_ep_rank=[32864, 32469, 32789, 32447, 32867, 32835, 33063, 32810] avg=32768.0000 max=33063 max_rank=6 min=32447 min_rank=3 imbalance_ratio=1.009003 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=8 tokens_per_ep_rank=[32501, 32704, 32850, 32380, 33151, 32781, 33034, 32743] avg=32768.0000 max=33151 max_rank=4 min=32380 min_rank=3 imbalance_ratio=1.011688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=10 tokens_per_ep_rank=[32864, 32469, 32789, 32447, 32867, 32835, 33063, 32810] avg=32768.0000 max=33063 max_rank=6 min=32447 min_rank=3 imbalance_ratio=1.009003 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=3 tokens_per_ep_rank=[33098, 32596, 32727, 32724, 32531, 32963, 32718, 32787] avg=32768.0000 max=33098 max_rank=0 min=32531 min_rank=4 imbalance_ratio=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=9 tokens_per_ep_rank=[32565, 32402, 32881, 32878, 32730, 32846, 33023, 32819] avg=32768.0000 max=33023 max_rank=6 min=32402 min_rank=1 imbalance_ratio=1.007782 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=8 tokens_per_ep_rank=[32501, 32704, 32850, 32380, 33151, 32781, 33034, 32743] avg=32768.0000 max=33151 max_rank=4 min=32380 min_rank=3 imbalance_ratio=1.011688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=8 tokens_per_ep_rank=[32501, 32704, 32850, 32435, 33096, 32781, 33034, 32743] avg=32768.0000 max=33096 max_rank=4 min=32435 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=55 max_load_reduction_pct=0.001659 moved_to_replicas=55 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=4 tokens_per_ep_rank=[32788, 32807, 32709, 33198, 32598, 32731, 32691, 32622] avg=32768.0000 max=33198 max_rank=3 min=32598 min_rank=4 imbalance_ratio=1.013123 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=9 tokens_per_ep_rank=[32565, 32402, 32881, 32878, 32730, 32846, 33023, 32819] avg=32768.0000 max=33023 max_rank=6 min=32402 min_rank=1 imbalance_ratio=1.007782 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=3 tokens_per_ep_rank=[33096, 32596, 32727, 32724, 32533, 32963, 32718, 32787] avg=32768.0000 max=33096 max_rank=0 min=32533 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=2 max_load_reduction_pct=0.000060 moved_to_replicas=2 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=8 tokens_per_ep_rank=[32501, 32704, 32850, 32435, 33096, 32781, 33034, 32743] avg=32768.0000 max=33096 max_rank=4 min=32435 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=55 max_load_reduction_pct=0.001659 moved_to_replicas=55 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=9 tokens_per_ep_rank=[32565, 32402, 32881, 32878, 32730, 32846, 33023, 32819] avg=32768.0000 max=33023 max_rank=6 min=32402 min_rank=1 imbalance_ratio=1.007782 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=4 tokens_per_ep_rank=[32788, 32807, 32709, 33096, 32700, 32731, 32691, 32622] avg=32768.0000 max=33096 max_rank=3 min=32622 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=102 max_load_reduction_pct=0.003072 moved_to_replicas=102 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=10 tokens_per_ep_rank=[32864, 32469, 32789, 32447, 32867, 32835, 33063, 32810] avg=32768.0000 max=33063 max_rank=6 min=32447 min_rank=3 imbalance_ratio=1.009003 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=9 tokens_per_ep_rank=[32565, 32402, 32881, 32878, 32730, 32846, 33023, 32819] avg=32768.0000 max=33023 max_rank=6 min=32402 min_rank=1 imbalance_ratio=1.007782 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=4 tokens_per_ep_rank=[32788, 32807, 32709, 33198, 32598, 32731, 32691, 32622] avg=32768.0000 max=33198 max_rank=3 min=32598 min_rank=4 imbalance_ratio=1.013123 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=10 tokens_per_ep_rank=[32864, 32469, 32789, 32447, 32867, 32835, 33063, 32810] avg=32768.0000 max=33063 max_rank=6 min=32447 min_rank=3 imbalance_ratio=1.009003 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=9 tokens_per_ep_rank=[32565, 32402, 32881, 32878, 32730, 32846, 33023, 32819] avg=32768.0000 max=33023 max_rank=6 min=32402 min_rank=1 imbalance_ratio=1.007782 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=9 tokens_per_ep_rank=[32565, 32402, 32881, 32878, 32730, 32846, 33023, 32819] avg=32768.0000 max=33023 max_rank=6 min=32402 min_rank=1 imbalance_ratio=1.007782 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=10 tokens_per_ep_rank=[32864, 32469, 32789, 32447, 32867, 32835, 33063, 32810] avg=32768.0000 max=33063 max_rank=6 min=32447 min_rank=3 imbalance_ratio=1.009003 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=5 tokens_per_ep_rank=[32664, 32868, 32891, 32965, 32683, 32725, 32758, 32590] avg=32768.0000 max=32965 max_rank=3 min=32590 min_rank=7 imbalance_ratio=1.006012 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=4 tokens_per_ep_rank=[32788, 32807, 32709, 33096, 32700, 32731, 32691, 32622] avg=32768.0000 max=33096 max_rank=3 min=32622 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=102 max_load_reduction_pct=0.003072 moved_to_replicas=102 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=9 tokens_per_ep_rank=[32565, 32402, 32881, 32878, 32730, 32846, 33023, 32819] avg=32768.0000 max=33023 max_rank=6 min=32402 min_rank=1 imbalance_ratio=1.007782 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=10 tokens_per_ep_rank=[32864, 32469, 32789, 32447, 32867, 32835, 33063, 32810] avg=32768.0000 max=33063 max_rank=6 min=32447 min_rank=3 imbalance_ratio=1.009003 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=5 tokens_per_ep_rank=[32664, 32868, 32891, 32965, 32683, 32725, 32758, 32590] avg=32768.0000 max=32965 max_rank=3 min=32590 min_rank=7 imbalance_ratio=1.006012 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=10 tokens_per_ep_rank=[32864, 32469, 32789, 32447, 32867, 32835, 33063, 32810] avg=32768.0000 max=33063 max_rank=6 min=32447 min_rank=3 imbalance_ratio=1.009003 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=5 tokens_per_ep_rank=[32664, 32868, 32891, 32965, 32683, 32725, 32758, 32590] avg=32768.0000 max=32965 max_rank=3 min=32590 min_rank=7 imbalance_ratio=1.006012 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=10 tokens_per_ep_rank=[32864, 32469, 32789, 32447, 32867, 32835, 33063, 32810] avg=32768.0000 max=33063 max_rank=6 min=32447 min_rank=3 imbalance_ratio=1.009003 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=10 tokens_per_ep_rank=[32864, 32469, 32789, 32447, 32867, 32835, 33063, 32810] avg=32768.0000 max=33063 max_rank=6 min=32447 min_rank=3 imbalance_ratio=1.009003 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=6 tokens_per_ep_rank=[32958, 32765, 32806, 32999, 32681, 32822, 32673, 32440] avg=32768.0000 max=32999 max_rank=3 min=32440 min_rank=7 imbalance_ratio=1.007050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=5 tokens_per_ep_rank=[32664, 32868, 32891, 32965, 32683, 32725, 32758, 32590] avg=32768.0000 max=32965 max_rank=3 min=32590 min_rank=7 imbalance_ratio=1.006012 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=0 records=10 total_expected_recv=328141 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76032 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=0 records=10 total_expected_recv=328040 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76800 peak_demand=32963 required_capacity=1.005951 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=10 tokens_per_ep_rank=[32864, 32469, 32789, 32447, 32867, 32835, 33063, 32810] avg=32768.0000 max=33063 max_rank=6 min=32447 min_rank=3 imbalance_ratio=1.009003 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=6 tokens_per_ep_rank=[32958, 32765, 32806, 32999, 32681, 32822, 32673, 32440] avg=32768.0000 max=32999 max_rank=3 min=32440 min_rank=7 imbalance_ratio=1.007050 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=6 tokens_per_ep_rank=[32958, 32765, 32806, 32999, 32681, 32822, 32673, 32440] avg=32768.0000 max=32999 max_rank=3 min=32440 min_rank=7 imbalance_ratio=1.007050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=7 tokens_per_ep_rank=[32994, 32777, 32775, 32427, 32642, 32908, 32812, 32809] avg=32768.0000 max=32994 max_rank=0 min=32427 min_rank=3 imbalance_ratio=1.006897 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=6 tokens_per_ep_rank=[32958, 32765, 32806, 32999, 32681, 32822, 32673, 32440] avg=32768.0000 max=32999 max_rank=3 min=32440 min_rank=7 imbalance_ratio=1.007050 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=0 records=10 total_expected_recv=327040 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=77440 peak_demand=32819 required_capacity=1.001556 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=7 tokens_per_ep_rank=[32994, 32777, 32775, 32427, 32642, 32908, 32812, 32809] avg=32768.0000 max=32994 max_rank=0 min=32427 min_rank=3 imbalance_ratio=1.006897 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=7 tokens_per_ep_rank=[32994, 32777, 32775, 32427, 32642, 32908, 32812, 32809] avg=32768.0000 max=32994 max_rank=0 min=32427 min_rank=3 imbalance_ratio=1.006897 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=8 tokens_per_ep_rank=[32501, 32704, 32850, 32380, 33151, 32781, 33034, 32743] avg=32768.0000 max=33151 max_rank=4 min=32380 min_rank=3 imbalance_ratio=1.011688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=0 records=10 total_expected_recv=326929 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=77184 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=7 tokens_per_ep_rank=[32994, 32777, 32775, 32427, 32642, 32908, 32812, 32809] avg=32768.0000 max=32994 max_rank=0 min=32427 min_rank=3 imbalance_ratio=1.006897 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=8 tokens_per_ep_rank=[32501, 32704, 32850, 32435, 33096, 32781, 33034, 32743] avg=32768.0000 max=33096 max_rank=4 min=32435 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=55 max_load_reduction_pct=0.001659 moved_to_replicas=55 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=0 records=10 total_expected_recv=327931 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76544 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=0 records=10 total_expected_recv=326982 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=77568 peak_demand=32868 required_capacity=1.003052 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=8 tokens_per_ep_rank=[32501, 32704, 32850, 32380, 33151, 32781, 33034, 32743] avg=32768.0000 max=33151 max_rank=4 min=32380 min_rank=3 imbalance_ratio=1.011688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=9 tokens_per_ep_rank=[32565, 32402, 32881, 32878, 32730, 32846, 33023, 32819] avg=32768.0000 max=33023 max_rank=6 min=32402 min_rank=1 imbalance_ratio=1.007782 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=8 tokens_per_ep_rank=[32501, 32704, 32850, 32435, 33096, 32781, 33034, 32743] avg=32768.0000 max=33096 max_rank=4 min=32435 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=55 max_load_reduction_pct=0.001659 moved_to_replicas=55 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=9 tokens_per_ep_rank=[32565, 32402, 32881, 32878, 32730, 32846, 33023, 32819] avg=32768.0000 max=33023 max_rank=6 min=32402 min_rank=1 imbalance_ratio=1.007782 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=9 tokens_per_ep_rank=[32565, 32402, 32881, 32878, 32730, 32846, 33023, 32819] avg=32768.0000 max=33023 max_rank=6 min=32402 min_rank=1 imbalance_ratio=1.007782 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=10 tokens_per_ep_rank=[32864, 32469, 32789, 32447, 32867, 32835, 33063, 32810] avg=32768.0000 max=33063 max_rank=6 min=32447 min_rank=3 imbalance_ratio=1.009003 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=9 tokens_per_ep_rank=[32565, 32402, 32881, 32878, 32730, 32846, 33023, 32819] avg=32768.0000 max=33023 max_rank=6 min=32402 min_rank=1 imbalance_ratio=1.007782 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=10 tokens_per_ep_rank=[32864, 32469, 32789, 32447, 32867, 32835, 33063, 32810] avg=32768.0000 max=33063 max_rank=6 min=32447 min_rank=3 imbalance_ratio=1.009003 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=10 tokens_per_ep_rank=[32864, 32469, 32789, 32447, 32867, 32835, 33063, 32810] avg=32768.0000 max=33063 max_rank=6 min=32447 min_rank=3 imbalance_ratio=1.009003 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=10 tokens_per_ep_rank=[32864, 32469, 32789, 32447, 32867, 32835, 33063, 32810] avg=32768.0000 max=33063 max_rank=6 min=32447 min_rank=3 imbalance_ratio=1.009003 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=0 records=10 total_expected_recv=328434 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76032 peak_demand=33063 required_capacity=1.009003 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=0 records=10 total_expected_recv=327943 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76288 peak_demand=32891 required_capacity=1.003754 + [2026-08-26 15:53:11] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 26090.5 | throughput per GPU (TFLOP/s/GPU): 4.7 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.098005E+01 | loss scale: 1.0 | grad norm: 29.745 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 33408 | min safe capacity: 1.0195 | +Number of parameters in transformer block in billions: 20.11 +Number of parameters in embedding layers in billions: 0.36 +Total number of parameters in billions: 20.47 +Number of parameters in most loaded shard in billions: 20.4724 +Theoretical memory footprints: weight and optimizer=146430.14 MB +[Rank 0] (after 1 iterations) memory (MB) | allocated: 64253.6591796875 | max allocated: 66328.82763671875 | reserved: 73058.0 | max reserved: 73058.0 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=0 calls=1 budget=40960 expected_recv=32739 expected_overflow=0 exact_recv_per_rank=[32779, 32819, 32738, 32617, 32873, 32739, 33011, 32568] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=0 calls=1 budget=40960 expected_recv=33011 expected_overflow=0 exact_recv_per_rank=[32779, 32819, 32738, 32617, 32873, 32739, 33011, 32568] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=0 calls=1 budget=40960 expected_recv=32873 expected_overflow=0 exact_recv_per_rank=[32779, 32819, 32738, 32617, 32873, 32739, 33011, 32568] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=0 calls=1 budget=40960 expected_recv=32738 expected_overflow=0 exact_recv_per_rank=[32779, 32819, 32738, 32617, 32873, 32739, 33011, 32568] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=0 calls=1 budget=40960 expected_recv=32568 expected_overflow=0 exact_recv_per_rank=[32779, 32819, 32738, 32617, 32873, 32739, 33011, 32568] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=0 calls=1 budget=40960 expected_recv=32617 expected_overflow=0 exact_recv_per_rank=[32779, 32819, 32738, 32617, 32873, 32739, 33011, 32568] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=0 calls=1 budget=40960 expected_recv=32739 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=0 calls=1 budget=40960 expected_recv=32819 expected_overflow=0 exact_recv_per_rank=[32779, 32819, 32738, 32617, 32873, 32739, 33011, 32568] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=0 calls=1 budget=40960 expected_recv=33011 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=0 calls=1 budget=40960 expected_recv=32873 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=0 calls=1 budget=40960 expected_recv=32738 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=0 calls=1 budget=40960 expected_recv=32568 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=0 calls=1 budget=40960 expected_recv=32779 expected_overflow=0 exact_recv_per_rank=[32779, 32819, 32738, 32617, 32873, 32739, 33011, 32568] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=0 calls=1 budget=40960 expected_recv=32819 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=0 calls=1 budget=40960 expected_recv=32617 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=0 calls=1 budget=40960 expected_recv=32779 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=1 calls=1 budget=40960 expected_recv=32970 expected_overflow=0 exact_recv_per_rank=[32753, 32903, 32783, 32569, 32716, 32970, 32678, 32772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=1 calls=1 budget=40960 expected_recv=32970 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=1 calls=1 budget=40960 expected_recv=32678 expected_overflow=0 exact_recv_per_rank=[32753, 32903, 32783, 32569, 32716, 32970, 32678, 32772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=1 calls=1 budget=40960 expected_recv=32716 expected_overflow=0 exact_recv_per_rank=[32753, 32903, 32783, 32569, 32716, 32970, 32678, 32772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=1 calls=1 budget=40960 expected_recv=32783 expected_overflow=0 exact_recv_per_rank=[32753, 32903, 32783, 32569, 32716, 32970, 32678, 32772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=1 calls=1 budget=40960 expected_recv=32678 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=1 calls=1 budget=40960 expected_recv=32772 expected_overflow=0 exact_recv_per_rank=[32753, 32903, 32783, 32569, 32716, 32970, 32678, 32772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=1 calls=1 budget=40960 expected_recv=32716 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=1 calls=1 budget=40960 expected_recv=32569 expected_overflow=0 exact_recv_per_rank=[32753, 32903, 32783, 32569, 32716, 32970, 32678, 32772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=1 calls=1 budget=40960 expected_recv=32783 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=1 calls=1 budget=40960 expected_recv=32903 expected_overflow=0 exact_recv_per_rank=[32753, 32903, 32783, 32569, 32716, 32970, 32678, 32772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=1 calls=1 budget=40960 expected_recv=32772 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=1 calls=1 budget=40960 expected_recv=32753 expected_overflow=0 exact_recv_per_rank=[32753, 32903, 32783, 32569, 32716, 32970, 32678, 32772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=1 calls=1 budget=40960 expected_recv=32569 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=2 calls=1 budget=40960 expected_recv=32838 expected_overflow=0 exact_recv_per_rank=[32577, 33005, 32812, 32747, 32773, 32838, 32700, 32692] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=1 calls=1 budget=40960 expected_recv=32903 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=1 calls=1 budget=40960 expected_recv=32753 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=2 calls=1 budget=40960 expected_recv=32838 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=2 calls=1 budget=40960 expected_recv=32700 expected_overflow=0 exact_recv_per_rank=[32577, 33005, 32812, 32747, 32773, 32838, 32700, 32692] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=2 calls=1 budget=40960 expected_recv=32773 expected_overflow=0 exact_recv_per_rank=[32577, 33005, 32812, 32747, 32773, 32838, 32700, 32692] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=2 calls=1 budget=40960 expected_recv=32812 expected_overflow=0 exact_recv_per_rank=[32577, 33005, 32812, 32747, 32773, 32838, 32700, 32692] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=2 calls=1 budget=40960 expected_recv=32773 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=2 calls=1 budget=40960 expected_recv=32692 expected_overflow=0 exact_recv_per_rank=[32577, 33005, 32812, 32747, 32773, 32838, 32700, 32692] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=2 calls=1 budget=40960 expected_recv=32700 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=2 calls=1 budget=40960 expected_recv=32747 expected_overflow=0 exact_recv_per_rank=[32577, 33005, 32812, 32747, 32773, 32838, 32700, 32692] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=2 calls=1 budget=40960 expected_recv=32812 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=2 calls=1 budget=40960 expected_recv=33005 expected_overflow=0 exact_recv_per_rank=[32577, 33005, 32812, 32747, 32773, 32838, 32700, 32692] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=2 calls=1 budget=40960 expected_recv=32577 expected_overflow=0 exact_recv_per_rank=[32577, 33005, 32812, 32747, 32773, 32838, 32700, 32692] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=3 calls=1 budget=40960 expected_recv=32755 expected_overflow=0 exact_recv_per_rank=[33087, 32827, 32715, 32663, 32850, 32755, 32635, 32612] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=2 calls=1 budget=40960 expected_recv=32692 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=2 calls=1 budget=40960 expected_recv=32747 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=2 calls=1 budget=40960 expected_recv=33005 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=2 calls=1 budget=40960 expected_recv=32577 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=3 calls=1 budget=40960 expected_recv=32755 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=3 calls=1 budget=40960 expected_recv=32850 expected_overflow=0 exact_recv_per_rank=[33087, 32827, 32715, 32663, 32850, 32755, 32635, 32612] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=3 calls=1 budget=40960 expected_recv=32635 expected_overflow=0 exact_recv_per_rank=[33087, 32827, 32715, 32663, 32850, 32755, 32635, 32612] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=3 calls=1 budget=40960 expected_recv=32850 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=3 calls=1 budget=40960 expected_recv=32715 expected_overflow=0 exact_recv_per_rank=[33087, 32827, 32715, 32663, 32850, 32755, 32635, 32612] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=3 calls=1 budget=40960 expected_recv=32612 expected_overflow=0 exact_recv_per_rank=[33087, 32827, 32715, 32663, 32850, 32755, 32635, 32612] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=3 calls=1 budget=40960 expected_recv=32635 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=3 calls=1 budget=40960 expected_recv=32663 expected_overflow=0 exact_recv_per_rank=[33087, 32827, 32715, 32663, 32850, 32755, 32635, 32612] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=3 calls=1 budget=40960 expected_recv=32827 expected_overflow=0 exact_recv_per_rank=[33087, 32827, 32715, 32663, 32850, 32755, 32635, 32612] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=3 calls=1 budget=40960 expected_recv=33087 expected_overflow=0 exact_recv_per_rank=[33087, 32827, 32715, 32663, 32850, 32755, 32635, 32612] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=3 calls=1 budget=40960 expected_recv=32715 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=4 calls=1 budget=40960 expected_recv=32649 expected_overflow=0 exact_recv_per_rank=[32739, 32790, 32942, 32691, 32709, 32649, 32596, 33028] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=3 calls=1 budget=40960 expected_recv=32612 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=3 calls=1 budget=40960 expected_recv=32663 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=3 calls=1 budget=40960 expected_recv=32827 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=3 calls=1 budget=40960 expected_recv=33087 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=4 calls=1 budget=40960 expected_recv=32649 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=4 calls=1 budget=40960 expected_recv=32709 expected_overflow=0 exact_recv_per_rank=[32739, 32790, 32942, 32691, 32709, 32649, 32596, 33028] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=4 calls=1 budget=40960 expected_recv=32709 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=4 calls=1 budget=40960 expected_recv=32596 expected_overflow=0 exact_recv_per_rank=[32739, 32790, 32942, 32691, 32709, 32649, 32596, 33028] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=4 calls=1 budget=40960 expected_recv=32942 expected_overflow=0 exact_recv_per_rank=[32739, 32790, 32942, 32691, 32709, 32649, 32596, 33028] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=4 calls=1 budget=40960 expected_recv=32596 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=4 calls=1 budget=40960 expected_recv=33028 expected_overflow=0 exact_recv_per_rank=[32739, 32790, 32942, 32691, 32709, 32649, 32596, 33028] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=4 calls=1 budget=40960 expected_recv=32942 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=4 calls=1 budget=40960 expected_recv=32739 expected_overflow=0 exact_recv_per_rank=[32739, 32790, 32942, 32691, 32709, 32649, 32596, 33028] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=4 calls=1 budget=40960 expected_recv=33028 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=4 calls=1 budget=40960 expected_recv=32691 expected_overflow=0 exact_recv_per_rank=[32739, 32790, 32942, 32691, 32709, 32649, 32596, 33028] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=4 calls=1 budget=40960 expected_recv=32790 expected_overflow=0 exact_recv_per_rank=[32739, 32790, 32942, 32691, 32709, 32649, 32596, 33028] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=4 calls=1 budget=40960 expected_recv=32739 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=5 calls=1 budget=40960 expected_recv=32764 expected_overflow=0 exact_recv_per_rank=[32466, 33017, 32775, 32700, 32723, 32764, 32795, 32904] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=4 calls=1 budget=40960 expected_recv=32691 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=4 calls=1 budget=40960 expected_recv=32790 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=5 calls=1 budget=40960 expected_recv=32723 expected_overflow=0 exact_recv_per_rank=[32466, 33017, 32775, 32700, 32723, 32764, 32795, 32904] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=5 calls=1 budget=40960 expected_recv=32764 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=5 calls=1 budget=40960 expected_recv=32723 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=5 calls=1 budget=40960 expected_recv=32795 expected_overflow=0 exact_recv_per_rank=[32466, 33017, 32775, 32700, 32723, 32764, 32795, 32904] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=5 calls=1 budget=40960 expected_recv=32795 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=5 calls=1 budget=40960 expected_recv=32775 expected_overflow=0 exact_recv_per_rank=[32466, 33017, 32775, 32700, 32723, 32764, 32795, 32904] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=5 calls=1 budget=40960 expected_recv=32904 expected_overflow=0 exact_recv_per_rank=[32466, 33017, 32775, 32700, 32723, 32764, 32795, 32904] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=5 calls=1 budget=40960 expected_recv=32466 expected_overflow=0 exact_recv_per_rank=[32466, 33017, 32775, 32700, 32723, 32764, 32795, 32904] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=5 calls=1 budget=40960 expected_recv=32700 expected_overflow=0 exact_recv_per_rank=[32466, 33017, 32775, 32700, 32723, 32764, 32795, 32904] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=5 calls=1 budget=40960 expected_recv=32775 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=6 calls=1 budget=40960 expected_recv=32702 expected_overflow=0 exact_recv_per_rank=[33077, 32389, 32854, 33024, 33069, 32702, 32958, 32071] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=5 calls=1 budget=40960 expected_recv=32904 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=5 calls=1 budget=40960 expected_recv=33017 expected_overflow=0 exact_recv_per_rank=[32466, 33017, 32775, 32700, 32723, 32764, 32795, 32904] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=5 calls=1 budget=40960 expected_recv=32466 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=5 calls=1 budget=40960 expected_recv=32700 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=6 calls=1 budget=40960 expected_recv=33069 expected_overflow=0 exact_recv_per_rank=[33077, 32389, 32854, 33024, 33069, 32702, 32958, 32071] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=6 calls=1 budget=40960 expected_recv=32702 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=5 calls=1 budget=40960 expected_recv=33017 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=6 calls=1 budget=40960 expected_recv=33069 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=6 calls=1 budget=40960 expected_recv=32958 expected_overflow=0 exact_recv_per_rank=[33077, 32389, 32854, 33024, 33069, 32702, 32958, 32071] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=6 calls=1 budget=40960 expected_recv=32958 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=6 calls=1 budget=40960 expected_recv=32854 expected_overflow=0 exact_recv_per_rank=[33077, 32389, 32854, 33024, 33069, 32702, 32958, 32071] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=6 calls=1 budget=40960 expected_recv=32071 expected_overflow=0 exact_recv_per_rank=[33077, 32389, 32854, 33024, 33069, 32702, 32958, 32071] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=6 calls=1 budget=40960 expected_recv=33077 expected_overflow=0 exact_recv_per_rank=[33077, 32389, 32854, 33024, 33069, 32702, 32958, 32071] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=6 calls=1 budget=40960 expected_recv=33024 expected_overflow=0 exact_recv_per_rank=[33077, 32389, 32854, 33024, 33069, 32702, 32958, 32071] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=7 calls=1 budget=40960 expected_recv=32990 expected_overflow=0 exact_recv_per_rank=[32965, 32699, 32761, 32871, 32502, 32990, 32892, 32464] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=6 calls=1 budget=40960 expected_recv=32854 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=6 calls=1 budget=40960 expected_recv=32071 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=6 calls=1 budget=40960 expected_recv=33077 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=6 calls=1 budget=40960 expected_recv=32389 expected_overflow=0 exact_recv_per_rank=[33077, 32389, 32854, 33024, 33069, 32702, 32958, 32071] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=6 calls=1 budget=40960 expected_recv=33024 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=7 calls=1 budget=40960 expected_recv=32502 expected_overflow=0 exact_recv_per_rank=[32965, 32699, 32761, 32871, 32502, 32990, 32892, 32464] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=7 calls=1 budget=40960 expected_recv=32990 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=6 calls=1 budget=40960 expected_recv=32389 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=7 calls=1 budget=40960 expected_recv=32502 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=7 calls=1 budget=40960 expected_recv=32892 expected_overflow=0 exact_recv_per_rank=[32965, 32699, 32761, 32871, 32502, 32990, 32892, 32464] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=7 calls=1 budget=40960 expected_recv=32892 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=7 calls=1 budget=40960 expected_recv=32761 expected_overflow=0 exact_recv_per_rank=[32965, 32699, 32761, 32871, 32502, 32990, 32892, 32464] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=7 calls=1 budget=40960 expected_recv=32464 expected_overflow=0 exact_recv_per_rank=[32965, 32699, 32761, 32871, 32502, 32990, 32892, 32464] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=7 calls=1 budget=40960 expected_recv=32965 expected_overflow=0 exact_recv_per_rank=[32965, 32699, 32761, 32871, 32502, 32990, 32892, 32464] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=7 calls=1 budget=40960 expected_recv=32871 expected_overflow=0 exact_recv_per_rank=[32965, 32699, 32761, 32871, 32502, 32990, 32892, 32464] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=7 calls=1 budget=40960 expected_recv=32761 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=8 calls=1 budget=40960 expected_recv=32826 expected_overflow=0 exact_recv_per_rank=[32594, 32706, 32809, 32911, 32679, 32826, 32928, 32691] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=7 calls=1 budget=40960 expected_recv=32464 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=7 calls=1 budget=40960 expected_recv=32965 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=7 calls=1 budget=40960 expected_recv=32871 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=8 calls=1 budget=40960 expected_recv=32679 expected_overflow=0 exact_recv_per_rank=[32594, 32706, 32809, 32911, 32679, 32826, 32928, 32691] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=8 calls=1 budget=40960 expected_recv=32826 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=7 calls=1 budget=40960 expected_recv=32699 expected_overflow=0 exact_recv_per_rank=[32965, 32699, 32761, 32871, 32502, 32990, 32892, 32464] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=8 calls=1 budget=40960 expected_recv=32679 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=7 calls=1 budget=40960 expected_recv=32699 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=8 calls=1 budget=40960 expected_recv=32928 expected_overflow=0 exact_recv_per_rank=[32594, 32706, 32809, 32911, 32679, 32826, 32928, 32691] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=8 calls=1 budget=40960 expected_recv=32928 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=8 calls=1 budget=40960 expected_recv=32809 expected_overflow=0 exact_recv_per_rank=[32594, 32706, 32809, 32911, 32679, 32826, 32928, 32691] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=8 calls=1 budget=40960 expected_recv=32594 expected_overflow=0 exact_recv_per_rank=[32594, 32706, 32809, 32911, 32679, 32826, 32928, 32691] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=8 calls=1 budget=40960 expected_recv=32809 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=8 calls=1 budget=40960 expected_recv=32691 expected_overflow=0 exact_recv_per_rank=[32594, 32706, 32809, 32911, 32679, 32826, 32928, 32691] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=9 calls=1 budget=40960 expected_recv=32523 expected_overflow=0 exact_recv_per_rank=[32867, 32618, 32887, 32815, 32841, 32523, 32797, 32796] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=8 calls=1 budget=40960 expected_recv=32911 expected_overflow=0 exact_recv_per_rank=[32594, 32706, 32809, 32911, 32679, 32826, 32928, 32691] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=8 calls=1 budget=40960 expected_recv=32594 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=8 calls=1 budget=40960 expected_recv=32691 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=9 calls=1 budget=40960 expected_recv=32841 expected_overflow=0 exact_recv_per_rank=[32867, 32618, 32887, 32815, 32841, 32523, 32797, 32796] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=9 calls=1 budget=40960 expected_recv=32523 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=8 calls=1 budget=40960 expected_recv=32911 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=8 calls=1 budget=40960 expected_recv=32706 expected_overflow=0 exact_recv_per_rank=[32594, 32706, 32809, 32911, 32679, 32826, 32928, 32691] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=9 calls=1 budget=40960 expected_recv=32841 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=8 calls=1 budget=40960 expected_recv=32706 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=9 calls=1 budget=40960 expected_recv=32797 expected_overflow=0 exact_recv_per_rank=[32867, 32618, 32887, 32815, 32841, 32523, 32797, 32796] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=9 calls=1 budget=40960 expected_recv=32797 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=9 calls=1 budget=40960 expected_recv=32887 expected_overflow=0 exact_recv_per_rank=[32867, 32618, 32887, 32815, 32841, 32523, 32797, 32796] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=9 calls=1 budget=40960 expected_recv=32887 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=9 calls=1 budget=40960 expected_recv=32867 expected_overflow=0 exact_recv_per_rank=[32867, 32618, 32887, 32815, 32841, 32523, 32797, 32796] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=9 calls=1 budget=40960 expected_recv=32796 expected_overflow=0 exact_recv_per_rank=[32867, 32618, 32887, 32815, 32841, 32523, 32797, 32796] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=9 calls=1 budget=40960 expected_recv=32815 expected_overflow=0 exact_recv_per_rank=[32867, 32618, 32887, 32815, 32841, 32523, 32797, 32796] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=1 tokens_per_ep_rank=[32779, 32819, 32738, 32617, 32873, 32739, 33011, 32568] avg=32768.0000 max=33011 max_rank=6 min=32568 min_rank=7 imbalance_ratio=1.007416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=9 calls=1 budget=40960 expected_recv=32867 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=9 calls=1 budget=40960 expected_recv=32618 expected_overflow=0 exact_recv_per_rank=[32867, 32618, 32887, 32815, 32841, 32523, 32797, 32796] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=9 calls=1 budget=40960 expected_recv=32796 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=9 calls=1 budget=40960 expected_recv=32815 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=1 tokens_per_ep_rank=[32779, 32819, 32738, 32617, 32873, 32739, 33011, 32568] avg=32768.0000 max=33011 max_rank=6 min=32568 min_rank=7 imbalance_ratio=1.007416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=9 calls=1 budget=40960 expected_recv=32618 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=1 tokens_per_ep_rank=[32779, 32819, 32738, 32617, 32873, 32739, 33011, 32568] avg=32768.0000 max=33011 max_rank=6 min=32568 min_rank=7 imbalance_ratio=1.007416 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=1 tokens_per_ep_rank=[32779, 32819, 32738, 32617, 32873, 32739, 33011, 32568] avg=32768.0000 max=33011 max_rank=6 min=32568 min_rank=7 imbalance_ratio=1.007416 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=1 tokens_per_ep_rank=[32779, 32819, 32738, 32617, 32873, 32739, 33011, 32568] avg=32768.0000 max=33011 max_rank=6 min=32568 min_rank=7 imbalance_ratio=1.007416 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=1 tokens_per_ep_rank=[32779, 32819, 32738, 32617, 32873, 32739, 33011, 32568] avg=32768.0000 max=33011 max_rank=6 min=32568 min_rank=7 imbalance_ratio=1.007416 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=1 tokens_per_ep_rank=[32779, 32819, 32738, 32617, 32873, 32739, 33011, 32568] avg=32768.0000 max=33011 max_rank=6 min=32568 min_rank=7 imbalance_ratio=1.007416 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=1 tokens_per_ep_rank=[32779, 32819, 32738, 32617, 32873, 32739, 33011, 32568] avg=32768.0000 max=33011 max_rank=6 min=32568 min_rank=7 imbalance_ratio=1.007416 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=1 tokens_per_ep_rank=[32779, 32819, 32738, 32617, 32873, 32739, 33011, 32568] avg=32768.0000 max=33011 max_rank=6 min=32568 min_rank=7 imbalance_ratio=1.007416 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=1 tokens_per_ep_rank=[32779, 32819, 32738, 32617, 32873, 32739, 33011, 32568] avg=32768.0000 max=33011 max_rank=6 min=32568 min_rank=7 imbalance_ratio=1.007416 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=2 tokens_per_ep_rank=[32753, 32903, 32783, 32569, 32716, 32970, 32678, 32772] avg=32768.0000 max=32970 max_rank=5 min=32569 min_rank=3 imbalance_ratio=1.006165 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=1 tokens_per_ep_rank=[32779, 32819, 32738, 32617, 32873, 32739, 33011, 32568] avg=32768.0000 max=33011 max_rank=6 min=32568 min_rank=7 imbalance_ratio=1.007416 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=2 tokens_per_ep_rank=[32753, 32903, 32783, 32569, 32716, 32970, 32678, 32772] avg=32768.0000 max=32970 max_rank=5 min=32569 min_rank=3 imbalance_ratio=1.006165 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=1 tokens_per_ep_rank=[32779, 32819, 32738, 32617, 32873, 32739, 33011, 32568] avg=32768.0000 max=33011 max_rank=6 min=32568 min_rank=7 imbalance_ratio=1.007416 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=1 tokens_per_ep_rank=[32779, 32819, 32738, 32617, 32873, 32739, 33011, 32568] avg=32768.0000 max=33011 max_rank=6 min=32568 min_rank=7 imbalance_ratio=1.007416 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=2 tokens_per_ep_rank=[32753, 32903, 32783, 32569, 32716, 32970, 32678, 32772] avg=32768.0000 max=32970 max_rank=5 min=32569 min_rank=3 imbalance_ratio=1.006165 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=1 tokens_per_ep_rank=[32779, 32819, 32738, 32617, 32873, 32739, 33011, 32568] avg=32768.0000 max=33011 max_rank=6 min=32568 min_rank=7 imbalance_ratio=1.007416 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=1 tokens_per_ep_rank=[32779, 32819, 32738, 32617, 32873, 32739, 33011, 32568] avg=32768.0000 max=33011 max_rank=6 min=32568 min_rank=7 imbalance_ratio=1.007416 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=1 tokens_per_ep_rank=[32779, 32819, 32738, 32617, 32873, 32739, 33011, 32568] avg=32768.0000 max=33011 max_rank=6 min=32568 min_rank=7 imbalance_ratio=1.007416 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=2 tokens_per_ep_rank=[32753, 32903, 32783, 32569, 32716, 32970, 32678, 32772] avg=32768.0000 max=32970 max_rank=5 min=32569 min_rank=3 imbalance_ratio=1.006165 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=2 tokens_per_ep_rank=[32753, 32903, 32783, 32569, 32716, 32970, 32678, 32772] avg=32768.0000 max=32970 max_rank=5 min=32569 min_rank=3 imbalance_ratio=1.006165 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=2 tokens_per_ep_rank=[32753, 32903, 32783, 32569, 32716, 32970, 32678, 32772] avg=32768.0000 max=32970 max_rank=5 min=32569 min_rank=3 imbalance_ratio=1.006165 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=2 tokens_per_ep_rank=[32753, 32903, 32783, 32569, 32716, 32970, 32678, 32772] avg=32768.0000 max=32970 max_rank=5 min=32569 min_rank=3 imbalance_ratio=1.006165 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=3 tokens_per_ep_rank=[32577, 33005, 32812, 32747, 32773, 32838, 32700, 32692] avg=32768.0000 max=33005 max_rank=1 min=32577 min_rank=0 imbalance_ratio=1.007233 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=2 tokens_per_ep_rank=[32753, 32903, 32783, 32569, 32716, 32970, 32678, 32772] avg=32768.0000 max=32970 max_rank=5 min=32569 min_rank=3 imbalance_ratio=1.006165 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=2 tokens_per_ep_rank=[32753, 32903, 32783, 32569, 32716, 32970, 32678, 32772] avg=32768.0000 max=32970 max_rank=5 min=32569 min_rank=3 imbalance_ratio=1.006165 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=2 tokens_per_ep_rank=[32753, 32903, 32783, 32569, 32716, 32970, 32678, 32772] avg=32768.0000 max=32970 max_rank=5 min=32569 min_rank=3 imbalance_ratio=1.006165 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=3 tokens_per_ep_rank=[32577, 33005, 32812, 32747, 32773, 32838, 32700, 32692] avg=32768.0000 max=33005 max_rank=1 min=32577 min_rank=0 imbalance_ratio=1.007233 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=2 tokens_per_ep_rank=[32753, 32903, 32783, 32569, 32716, 32970, 32678, 32772] avg=32768.0000 max=32970 max_rank=5 min=32569 min_rank=3 imbalance_ratio=1.006165 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=2 tokens_per_ep_rank=[32753, 32903, 32783, 32569, 32716, 32970, 32678, 32772] avg=32768.0000 max=32970 max_rank=5 min=32569 min_rank=3 imbalance_ratio=1.006165 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=3 tokens_per_ep_rank=[32577, 33005, 32812, 32747, 32773, 32838, 32700, 32692] avg=32768.0000 max=33005 max_rank=1 min=32577 min_rank=0 imbalance_ratio=1.007233 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=2 tokens_per_ep_rank=[32753, 32903, 32783, 32569, 32716, 32970, 32678, 32772] avg=32768.0000 max=32970 max_rank=5 min=32569 min_rank=3 imbalance_ratio=1.006165 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=2 tokens_per_ep_rank=[32753, 32903, 32783, 32569, 32716, 32970, 32678, 32772] avg=32768.0000 max=32970 max_rank=5 min=32569 min_rank=3 imbalance_ratio=1.006165 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=2 tokens_per_ep_rank=[32753, 32903, 32783, 32569, 32716, 32970, 32678, 32772] avg=32768.0000 max=32970 max_rank=5 min=32569 min_rank=3 imbalance_ratio=1.006165 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=3 tokens_per_ep_rank=[32577, 33005, 32812, 32747, 32773, 32838, 32700, 32692] avg=32768.0000 max=33005 max_rank=1 min=32577 min_rank=0 imbalance_ratio=1.007233 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=3 tokens_per_ep_rank=[32577, 33005, 32812, 32747, 32773, 32838, 32700, 32692] avg=32768.0000 max=33005 max_rank=1 min=32577 min_rank=0 imbalance_ratio=1.007233 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=2 tokens_per_ep_rank=[32753, 32903, 32783, 32569, 32716, 32970, 32678, 32772] avg=32768.0000 max=32970 max_rank=5 min=32569 min_rank=3 imbalance_ratio=1.006165 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=3 tokens_per_ep_rank=[32577, 33005, 32812, 32747, 32773, 32838, 32700, 32692] avg=32768.0000 max=33005 max_rank=1 min=32577 min_rank=0 imbalance_ratio=1.007233 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=3 tokens_per_ep_rank=[32577, 33005, 32812, 32747, 32773, 32838, 32700, 32692] avg=32768.0000 max=33005 max_rank=1 min=32577 min_rank=0 imbalance_ratio=1.007233 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=4 tokens_per_ep_rank=[33087, 32827, 32715, 32663, 32850, 32755, 32635, 32612] avg=32768.0000 max=33087 max_rank=0 min=32612 min_rank=7 imbalance_ratio=1.009735 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=3 tokens_per_ep_rank=[32577, 33005, 32812, 32747, 32773, 32838, 32700, 32692] avg=32768.0000 max=33005 max_rank=1 min=32577 min_rank=0 imbalance_ratio=1.007233 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=3 tokens_per_ep_rank=[32577, 33005, 32812, 32747, 32773, 32838, 32700, 32692] avg=32768.0000 max=33005 max_rank=1 min=32577 min_rank=0 imbalance_ratio=1.007233 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=3 tokens_per_ep_rank=[32577, 33005, 32812, 32747, 32773, 32838, 32700, 32692] avg=32768.0000 max=33005 max_rank=1 min=32577 min_rank=0 imbalance_ratio=1.007233 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=4 tokens_per_ep_rank=[33087, 32827, 32715, 32663, 32850, 32755, 32635, 32612] avg=32768.0000 max=33087 max_rank=0 min=32612 min_rank=7 imbalance_ratio=1.009735 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=3 tokens_per_ep_rank=[32577, 33005, 32812, 32747, 32773, 32838, 32700, 32692] avg=32768.0000 max=33005 max_rank=1 min=32577 min_rank=0 imbalance_ratio=1.007233 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=3 tokens_per_ep_rank=[32577, 33005, 32812, 32747, 32773, 32838, 32700, 32692] avg=32768.0000 max=33005 max_rank=1 min=32577 min_rank=0 imbalance_ratio=1.007233 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=4 tokens_per_ep_rank=[33087, 32827, 32715, 32663, 32850, 32755, 32635, 32612] avg=32768.0000 max=33087 max_rank=0 min=32612 min_rank=7 imbalance_ratio=1.009735 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=3 tokens_per_ep_rank=[32577, 33005, 32812, 32747, 32773, 32838, 32700, 32692] avg=32768.0000 max=33005 max_rank=1 min=32577 min_rank=0 imbalance_ratio=1.007233 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=3 tokens_per_ep_rank=[32577, 33005, 32812, 32747, 32773, 32838, 32700, 32692] avg=32768.0000 max=33005 max_rank=1 min=32577 min_rank=0 imbalance_ratio=1.007233 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=4 tokens_per_ep_rank=[33087, 32827, 32715, 32663, 32850, 32755, 32635, 32612] avg=32768.0000 max=33087 max_rank=0 min=32612 min_rank=7 imbalance_ratio=1.009735 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=3 tokens_per_ep_rank=[32577, 33005, 32812, 32747, 32773, 32838, 32700, 32692] avg=32768.0000 max=33005 max_rank=1 min=32577 min_rank=0 imbalance_ratio=1.007233 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=4 tokens_per_ep_rank=[33087, 32827, 32715, 32663, 32850, 32755, 32635, 32612] avg=32768.0000 max=33087 max_rank=0 min=32612 min_rank=7 imbalance_ratio=1.009735 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=3 tokens_per_ep_rank=[32577, 33005, 32812, 32747, 32773, 32838, 32700, 32692] avg=32768.0000 max=33005 max_rank=1 min=32577 min_rank=0 imbalance_ratio=1.007233 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=4 tokens_per_ep_rank=[33087, 32827, 32715, 32663, 32850, 32755, 32635, 32612] avg=32768.0000 max=33087 max_rank=0 min=32612 min_rank=7 imbalance_ratio=1.009735 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=4 tokens_per_ep_rank=[33087, 32827, 32715, 32663, 32850, 32755, 32635, 32612] avg=32768.0000 max=33087 max_rank=0 min=32612 min_rank=7 imbalance_ratio=1.009735 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=5 tokens_per_ep_rank=[32739, 32790, 32942, 32691, 32709, 32649, 32596, 33028] avg=32768.0000 max=33028 max_rank=7 min=32596 min_rank=6 imbalance_ratio=1.007935 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=4 tokens_per_ep_rank=[33087, 32827, 32715, 32663, 32850, 32755, 32635, 32612] avg=32768.0000 max=33087 max_rank=0 min=32612 min_rank=7 imbalance_ratio=1.009735 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=4 tokens_per_ep_rank=[33087, 32827, 32715, 32663, 32850, 32755, 32635, 32612] avg=32768.0000 max=33087 max_rank=0 min=32612 min_rank=7 imbalance_ratio=1.009735 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=5 tokens_per_ep_rank=[32739, 32790, 32942, 32691, 32709, 32649, 32596, 33028] avg=32768.0000 max=33028 max_rank=7 min=32596 min_rank=6 imbalance_ratio=1.007935 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=4 tokens_per_ep_rank=[33087, 32827, 32715, 32663, 32850, 32755, 32635, 32612] avg=32768.0000 max=33087 max_rank=0 min=32612 min_rank=7 imbalance_ratio=1.009735 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=4 tokens_per_ep_rank=[33087, 32827, 32715, 32663, 32850, 32755, 32635, 32612] avg=32768.0000 max=33087 max_rank=0 min=32612 min_rank=7 imbalance_ratio=1.009735 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=4 tokens_per_ep_rank=[33087, 32827, 32715, 32663, 32850, 32755, 32635, 32612] avg=32768.0000 max=33087 max_rank=0 min=32612 min_rank=7 imbalance_ratio=1.009735 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=5 tokens_per_ep_rank=[32739, 32790, 32942, 32691, 32709, 32649, 32596, 33028] avg=32768.0000 max=33028 max_rank=7 min=32596 min_rank=6 imbalance_ratio=1.007935 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=4 tokens_per_ep_rank=[33087, 32827, 32715, 32663, 32850, 32755, 32635, 32612] avg=32768.0000 max=33087 max_rank=0 min=32612 min_rank=7 imbalance_ratio=1.009735 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=4 tokens_per_ep_rank=[33087, 32827, 32715, 32663, 32850, 32755, 32635, 32612] avg=32768.0000 max=33087 max_rank=0 min=32612 min_rank=7 imbalance_ratio=1.009735 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=5 tokens_per_ep_rank=[32739, 32790, 32942, 32691, 32709, 32649, 32596, 33028] avg=32768.0000 max=33028 max_rank=7 min=32596 min_rank=6 imbalance_ratio=1.007935 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=4 tokens_per_ep_rank=[33087, 32827, 32715, 32663, 32850, 32755, 32635, 32612] avg=32768.0000 max=33087 max_rank=0 min=32612 min_rank=7 imbalance_ratio=1.009735 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=4 tokens_per_ep_rank=[33087, 32827, 32715, 32663, 32850, 32755, 32635, 32612] avg=32768.0000 max=33087 max_rank=0 min=32612 min_rank=7 imbalance_ratio=1.009735 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=5 tokens_per_ep_rank=[32739, 32790, 32942, 32691, 32709, 32649, 32596, 33028] avg=32768.0000 max=33028 max_rank=7 min=32596 min_rank=6 imbalance_ratio=1.007935 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=6 tokens_per_ep_rank=[32466, 33017, 32775, 32700, 32723, 32764, 32795, 32904] avg=32768.0000 max=33017 max_rank=1 min=32466 min_rank=0 imbalance_ratio=1.007599 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=5 tokens_per_ep_rank=[32739, 32790, 32942, 32691, 32709, 32649, 32596, 33028] avg=32768.0000 max=33028 max_rank=7 min=32596 min_rank=6 imbalance_ratio=1.007935 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=5 tokens_per_ep_rank=[32739, 32790, 32942, 32691, 32709, 32649, 32596, 33028] avg=32768.0000 max=33028 max_rank=7 min=32596 min_rank=6 imbalance_ratio=1.007935 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=5 tokens_per_ep_rank=[32739, 32790, 32942, 32691, 32709, 32649, 32596, 33028] avg=32768.0000 max=33028 max_rank=7 min=32596 min_rank=6 imbalance_ratio=1.007935 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=5 tokens_per_ep_rank=[32739, 32790, 32942, 32691, 32709, 32649, 32596, 33028] avg=32768.0000 max=33028 max_rank=7 min=32596 min_rank=6 imbalance_ratio=1.007935 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=5 tokens_per_ep_rank=[32739, 32790, 32942, 32691, 32709, 32649, 32596, 33028] avg=32768.0000 max=33028 max_rank=7 min=32596 min_rank=6 imbalance_ratio=1.007935 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=5 tokens_per_ep_rank=[32739, 32790, 32942, 32691, 32709, 32649, 32596, 33028] avg=32768.0000 max=33028 max_rank=7 min=32596 min_rank=6 imbalance_ratio=1.007935 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=6 tokens_per_ep_rank=[32466, 33017, 32775, 32700, 32723, 32764, 32795, 32904] avg=32768.0000 max=33017 max_rank=1 min=32466 min_rank=0 imbalance_ratio=1.007599 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=5 tokens_per_ep_rank=[32739, 32790, 32942, 32691, 32709, 32649, 32596, 33028] avg=32768.0000 max=33028 max_rank=7 min=32596 min_rank=6 imbalance_ratio=1.007935 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=6 tokens_per_ep_rank=[32466, 33017, 32775, 32700, 32723, 32764, 32795, 32904] avg=32768.0000 max=33017 max_rank=1 min=32466 min_rank=0 imbalance_ratio=1.007599 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=5 tokens_per_ep_rank=[32739, 32790, 32942, 32691, 32709, 32649, 32596, 33028] avg=32768.0000 max=33028 max_rank=7 min=32596 min_rank=6 imbalance_ratio=1.007935 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=5 tokens_per_ep_rank=[32739, 32790, 32942, 32691, 32709, 32649, 32596, 33028] avg=32768.0000 max=33028 max_rank=7 min=32596 min_rank=6 imbalance_ratio=1.007935 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=6 tokens_per_ep_rank=[32466, 33017, 32775, 32700, 32723, 32764, 32795, 32904] avg=32768.0000 max=33017 max_rank=1 min=32466 min_rank=0 imbalance_ratio=1.007599 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=5 tokens_per_ep_rank=[32739, 32790, 32942, 32691, 32709, 32649, 32596, 33028] avg=32768.0000 max=33028 max_rank=7 min=32596 min_rank=6 imbalance_ratio=1.007935 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=5 tokens_per_ep_rank=[32739, 32790, 32942, 32691, 32709, 32649, 32596, 33028] avg=32768.0000 max=33028 max_rank=7 min=32596 min_rank=6 imbalance_ratio=1.007935 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=7 tokens_per_ep_rank=[33077, 32389, 32854, 33024, 33069, 32702, 32958, 32071] avg=32768.0000 max=33077 max_rank=0 min=32071 min_rank=7 imbalance_ratio=1.009430 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=6 tokens_per_ep_rank=[32466, 33017, 32775, 32700, 32723, 32764, 32795, 32904] avg=32768.0000 max=33017 max_rank=1 min=32466 min_rank=0 imbalance_ratio=1.007599 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=6 tokens_per_ep_rank=[32466, 33017, 32775, 32700, 32723, 32764, 32795, 32904] avg=32768.0000 max=33017 max_rank=1 min=32466 min_rank=0 imbalance_ratio=1.007599 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=6 tokens_per_ep_rank=[32466, 33017, 32775, 32700, 32723, 32764, 32795, 32904] avg=32768.0000 max=33017 max_rank=1 min=32466 min_rank=0 imbalance_ratio=1.007599 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=7 tokens_per_ep_rank=[33077, 32389, 32854, 33024, 33069, 32702, 32958, 32071] avg=32768.0000 max=33077 max_rank=0 min=32071 min_rank=7 imbalance_ratio=1.009430 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=6 tokens_per_ep_rank=[32466, 33017, 32775, 32700, 32723, 32764, 32795, 32904] avg=32768.0000 max=33017 max_rank=1 min=32466 min_rank=0 imbalance_ratio=1.007599 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=6 tokens_per_ep_rank=[32466, 33017, 32775, 32700, 32723, 32764, 32795, 32904] avg=32768.0000 max=33017 max_rank=1 min=32466 min_rank=0 imbalance_ratio=1.007599 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=6 tokens_per_ep_rank=[32466, 33017, 32775, 32700, 32723, 32764, 32795, 32904] avg=32768.0000 max=33017 max_rank=1 min=32466 min_rank=0 imbalance_ratio=1.007599 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=6 tokens_per_ep_rank=[32466, 33017, 32775, 32700, 32723, 32764, 32795, 32904] avg=32768.0000 max=33017 max_rank=1 min=32466 min_rank=0 imbalance_ratio=1.007599 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=6 tokens_per_ep_rank=[32466, 33017, 32775, 32700, 32723, 32764, 32795, 32904] avg=32768.0000 max=33017 max_rank=1 min=32466 min_rank=0 imbalance_ratio=1.007599 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=7 tokens_per_ep_rank=[33077, 32389, 32854, 33024, 33069, 32702, 32958, 32071] avg=32768.0000 max=33077 max_rank=0 min=32071 min_rank=7 imbalance_ratio=1.009430 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=6 tokens_per_ep_rank=[32466, 33017, 32775, 32700, 32723, 32764, 32795, 32904] avg=32768.0000 max=33017 max_rank=1 min=32466 min_rank=0 imbalance_ratio=1.007599 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=6 tokens_per_ep_rank=[32466, 33017, 32775, 32700, 32723, 32764, 32795, 32904] avg=32768.0000 max=33017 max_rank=1 min=32466 min_rank=0 imbalance_ratio=1.007599 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=7 tokens_per_ep_rank=[33077, 32389, 32854, 33024, 33069, 32702, 32958, 32071] avg=32768.0000 max=33077 max_rank=0 min=32071 min_rank=7 imbalance_ratio=1.009430 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=6 tokens_per_ep_rank=[32466, 33017, 32775, 32700, 32723, 32764, 32795, 32904] avg=32768.0000 max=33017 max_rank=1 min=32466 min_rank=0 imbalance_ratio=1.007599 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=6 tokens_per_ep_rank=[32466, 33017, 32775, 32700, 32723, 32764, 32795, 32904] avg=32768.0000 max=33017 max_rank=1 min=32466 min_rank=0 imbalance_ratio=1.007599 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=8 tokens_per_ep_rank=[32965, 32699, 32761, 32871, 32502, 32990, 32892, 32464] avg=32768.0000 max=32990 max_rank=5 min=32464 min_rank=7 imbalance_ratio=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=7 tokens_per_ep_rank=[33077, 32389, 32854, 33024, 33069, 32702, 32958, 32071] avg=32768.0000 max=33077 max_rank=0 min=32071 min_rank=7 imbalance_ratio=1.009430 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=7 tokens_per_ep_rank=[33077, 32389, 32854, 33024, 33069, 32702, 32958, 32071] avg=32768.0000 max=33077 max_rank=0 min=32071 min_rank=7 imbalance_ratio=1.009430 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=7 tokens_per_ep_rank=[33077, 32389, 32854, 33024, 33069, 32702, 32958, 32071] avg=32768.0000 max=33077 max_rank=0 min=32071 min_rank=7 imbalance_ratio=1.009430 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=8 tokens_per_ep_rank=[32965, 32699, 32761, 32871, 32502, 32990, 32892, 32464] avg=32768.0000 max=32990 max_rank=5 min=32464 min_rank=7 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=7 tokens_per_ep_rank=[33077, 32389, 32854, 33024, 33069, 32702, 32958, 32071] avg=32768.0000 max=33077 max_rank=0 min=32071 min_rank=7 imbalance_ratio=1.009430 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=7 tokens_per_ep_rank=[33077, 32389, 32854, 33024, 33069, 32702, 32958, 32071] avg=32768.0000 max=33077 max_rank=0 min=32071 min_rank=7 imbalance_ratio=1.009430 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=7 tokens_per_ep_rank=[33077, 32389, 32854, 33024, 33069, 32702, 32958, 32071] avg=32768.0000 max=33077 max_rank=0 min=32071 min_rank=7 imbalance_ratio=1.009430 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=7 tokens_per_ep_rank=[33077, 32389, 32854, 33024, 33069, 32702, 32958, 32071] avg=32768.0000 max=33077 max_rank=0 min=32071 min_rank=7 imbalance_ratio=1.009430 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=7 tokens_per_ep_rank=[33077, 32389, 32854, 33024, 33069, 32702, 32958, 32071] avg=32768.0000 max=33077 max_rank=0 min=32071 min_rank=7 imbalance_ratio=1.009430 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=8 tokens_per_ep_rank=[32965, 32699, 32761, 32871, 32502, 32990, 32892, 32464] avg=32768.0000 max=32990 max_rank=5 min=32464 min_rank=7 imbalance_ratio=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=8 tokens_per_ep_rank=[32965, 32699, 32761, 32871, 32502, 32990, 32892, 32464] avg=32768.0000 max=32990 max_rank=5 min=32464 min_rank=7 imbalance_ratio=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=7 tokens_per_ep_rank=[33077, 32389, 32854, 33024, 33069, 32702, 32958, 32071] avg=32768.0000 max=33077 max_rank=0 min=32071 min_rank=7 imbalance_ratio=1.009430 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=7 tokens_per_ep_rank=[33077, 32389, 32854, 33024, 33069, 32702, 32958, 32071] avg=32768.0000 max=33077 max_rank=0 min=32071 min_rank=7 imbalance_ratio=1.009430 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=7 tokens_per_ep_rank=[33077, 32389, 32854, 33024, 33069, 32702, 32958, 32071] avg=32768.0000 max=33077 max_rank=0 min=32071 min_rank=7 imbalance_ratio=1.009430 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=7 tokens_per_ep_rank=[33077, 32389, 32854, 33024, 33069, 32702, 32958, 32071] avg=32768.0000 max=33077 max_rank=0 min=32071 min_rank=7 imbalance_ratio=1.009430 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=9 tokens_per_ep_rank=[32594, 32706, 32809, 32911, 32679, 32826, 32928, 32691] avg=32768.0000 max=32928 max_rank=6 min=32594 min_rank=0 imbalance_ratio=1.004883 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=8 tokens_per_ep_rank=[32965, 32699, 32761, 32871, 32502, 32990, 32892, 32464] avg=32768.0000 max=32990 max_rank=5 min=32464 min_rank=7 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=8 tokens_per_ep_rank=[32965, 32699, 32761, 32871, 32502, 32990, 32892, 32464] avg=32768.0000 max=32990 max_rank=5 min=32464 min_rank=7 imbalance_ratio=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=8 tokens_per_ep_rank=[32965, 32699, 32761, 32871, 32502, 32990, 32892, 32464] avg=32768.0000 max=32990 max_rank=5 min=32464 min_rank=7 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=9 tokens_per_ep_rank=[32594, 32706, 32809, 32911, 32679, 32826, 32928, 32691] avg=32768.0000 max=32928 max_rank=6 min=32594 min_rank=0 imbalance_ratio=1.004883 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=8 tokens_per_ep_rank=[32965, 32699, 32761, 32871, 32502, 32990, 32892, 32464] avg=32768.0000 max=32990 max_rank=5 min=32464 min_rank=7 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=8 tokens_per_ep_rank=[32965, 32699, 32761, 32871, 32502, 32990, 32892, 32464] avg=32768.0000 max=32990 max_rank=5 min=32464 min_rank=7 imbalance_ratio=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=8 tokens_per_ep_rank=[32965, 32699, 32761, 32871, 32502, 32990, 32892, 32464] avg=32768.0000 max=32990 max_rank=5 min=32464 min_rank=7 imbalance_ratio=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=8 tokens_per_ep_rank=[32965, 32699, 32761, 32871, 32502, 32990, 32892, 32464] avg=32768.0000 max=32990 max_rank=5 min=32464 min_rank=7 imbalance_ratio=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=8 tokens_per_ep_rank=[32965, 32699, 32761, 32871, 32502, 32990, 32892, 32464] avg=32768.0000 max=32990 max_rank=5 min=32464 min_rank=7 imbalance_ratio=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=9 tokens_per_ep_rank=[32594, 32706, 32809, 32911, 32679, 32826, 32928, 32691] avg=32768.0000 max=32928 max_rank=6 min=32594 min_rank=0 imbalance_ratio=1.004883 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=9 tokens_per_ep_rank=[32594, 32706, 32809, 32911, 32679, 32826, 32928, 32691] avg=32768.0000 max=32928 max_rank=6 min=32594 min_rank=0 imbalance_ratio=1.004883 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=8 tokens_per_ep_rank=[32965, 32699, 32761, 32871, 32502, 32990, 32892, 32464] avg=32768.0000 max=32990 max_rank=5 min=32464 min_rank=7 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=8 tokens_per_ep_rank=[32965, 32699, 32761, 32871, 32502, 32990, 32892, 32464] avg=32768.0000 max=32990 max_rank=5 min=32464 min_rank=7 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=8 tokens_per_ep_rank=[32965, 32699, 32761, 32871, 32502, 32990, 32892, 32464] avg=32768.0000 max=32990 max_rank=5 min=32464 min_rank=7 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=10 tokens_per_ep_rank=[32867, 32618, 32887, 32815, 32841, 32523, 32797, 32796] avg=32768.0000 max=32887 max_rank=2 min=32523 min_rank=5 imbalance_ratio=1.003632 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=8 tokens_per_ep_rank=[32965, 32699, 32761, 32871, 32502, 32990, 32892, 32464] avg=32768.0000 max=32990 max_rank=5 min=32464 min_rank=7 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=9 tokens_per_ep_rank=[32594, 32706, 32809, 32911, 32679, 32826, 32928, 32691] avg=32768.0000 max=32928 max_rank=6 min=32594 min_rank=0 imbalance_ratio=1.004883 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=9 tokens_per_ep_rank=[32594, 32706, 32809, 32911, 32679, 32826, 32928, 32691] avg=32768.0000 max=32928 max_rank=6 min=32594 min_rank=0 imbalance_ratio=1.004883 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=9 tokens_per_ep_rank=[32594, 32706, 32809, 32911, 32679, 32826, 32928, 32691] avg=32768.0000 max=32928 max_rank=6 min=32594 min_rank=0 imbalance_ratio=1.004883 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=10 tokens_per_ep_rank=[32867, 32618, 32887, 32815, 32841, 32523, 32797, 32796] avg=32768.0000 max=32887 max_rank=2 min=32523 min_rank=5 imbalance_ratio=1.003632 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=9 tokens_per_ep_rank=[32594, 32706, 32809, 32911, 32679, 32826, 32928, 32691] avg=32768.0000 max=32928 max_rank=6 min=32594 min_rank=0 imbalance_ratio=1.004883 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=9 tokens_per_ep_rank=[32594, 32706, 32809, 32911, 32679, 32826, 32928, 32691] avg=32768.0000 max=32928 max_rank=6 min=32594 min_rank=0 imbalance_ratio=1.004883 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=9 tokens_per_ep_rank=[32594, 32706, 32809, 32911, 32679, 32826, 32928, 32691] avg=32768.0000 max=32928 max_rank=6 min=32594 min_rank=0 imbalance_ratio=1.004883 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=9 tokens_per_ep_rank=[32594, 32706, 32809, 32911, 32679, 32826, 32928, 32691] avg=32768.0000 max=32928 max_rank=6 min=32594 min_rank=0 imbalance_ratio=1.004883 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=9 tokens_per_ep_rank=[32594, 32706, 32809, 32911, 32679, 32826, 32928, 32691] avg=32768.0000 max=32928 max_rank=6 min=32594 min_rank=0 imbalance_ratio=1.004883 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=10 tokens_per_ep_rank=[32867, 32618, 32887, 32815, 32841, 32523, 32797, 32796] avg=32768.0000 max=32887 max_rank=2 min=32523 min_rank=5 imbalance_ratio=1.003632 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=10 tokens_per_ep_rank=[32867, 32618, 32887, 32815, 32841, 32523, 32797, 32796] avg=32768.0000 max=32887 max_rank=2 min=32523 min_rank=5 imbalance_ratio=1.003632 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=9 tokens_per_ep_rank=[32594, 32706, 32809, 32911, 32679, 32826, 32928, 32691] avg=32768.0000 max=32928 max_rank=6 min=32594 min_rank=0 imbalance_ratio=1.004883 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=9 tokens_per_ep_rank=[32594, 32706, 32809, 32911, 32679, 32826, 32928, 32691] avg=32768.0000 max=32928 max_rank=6 min=32594 min_rank=0 imbalance_ratio=1.004883 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=9 tokens_per_ep_rank=[32594, 32706, 32809, 32911, 32679, 32826, 32928, 32691] avg=32768.0000 max=32928 max_rank=6 min=32594 min_rank=0 imbalance_ratio=1.004883 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=10 tokens_per_ep_rank=[32867, 32618, 32887, 32815, 32841, 32523, 32797, 32796] avg=32768.0000 max=32887 max_rank=2 min=32523 min_rank=5 imbalance_ratio=1.003632 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=9 tokens_per_ep_rank=[32594, 32706, 32809, 32911, 32679, 32826, 32928, 32691] avg=32768.0000 max=32928 max_rank=6 min=32594 min_rank=0 imbalance_ratio=1.004883 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=10 tokens_per_ep_rank=[32867, 32618, 32887, 32815, 32841, 32523, 32797, 32796] avg=32768.0000 max=32887 max_rank=2 min=32523 min_rank=5 imbalance_ratio=1.003632 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=10 tokens_per_ep_rank=[32867, 32618, 32887, 32815, 32841, 32523, 32797, 32796] avg=32768.0000 max=32887 max_rank=2 min=32523 min_rank=5 imbalance_ratio=1.003632 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=10 tokens_per_ep_rank=[32867, 32618, 32887, 32815, 32841, 32523, 32797, 32796] avg=32768.0000 max=32887 max_rank=2 min=32523 min_rank=5 imbalance_ratio=1.003632 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=10 tokens_per_ep_rank=[32867, 32618, 32887, 32815, 32841, 32523, 32797, 32796] avg=32768.0000 max=32887 max_rank=2 min=32523 min_rank=5 imbalance_ratio=1.003632 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=10 tokens_per_ep_rank=[32867, 32618, 32887, 32815, 32841, 32523, 32797, 32796] avg=32768.0000 max=32887 max_rank=2 min=32523 min_rank=5 imbalance_ratio=1.003632 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=10 tokens_per_ep_rank=[32867, 32618, 32887, 32815, 32841, 32523, 32797, 32796] avg=32768.0000 max=32887 max_rank=2 min=32523 min_rank=5 imbalance_ratio=1.003632 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=10 tokens_per_ep_rank=[32867, 32618, 32887, 32815, 32841, 32523, 32797, 32796] avg=32768.0000 max=32887 max_rank=2 min=32523 min_rank=5 imbalance_ratio=1.003632 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=10 tokens_per_ep_rank=[32867, 32618, 32887, 32815, 32841, 32523, 32797, 32796] avg=32768.0000 max=32887 max_rank=2 min=32523 min_rank=5 imbalance_ratio=1.003632 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=10 tokens_per_ep_rank=[32867, 32618, 32887, 32815, 32841, 32523, 32797, 32796] avg=32768.0000 max=32887 max_rank=2 min=32523 min_rank=5 imbalance_ratio=1.003632 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=10 tokens_per_ep_rank=[32867, 32618, 32887, 32815, 32841, 32523, 32797, 32796] avg=32768.0000 max=32887 max_rank=2 min=32523 min_rank=5 imbalance_ratio=1.003632 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=10 tokens_per_ep_rank=[32867, 32618, 32887, 32815, 32841, 32523, 32797, 32796] avg=32768.0000 max=32887 max_rank=2 min=32523 min_rank=5 imbalance_ratio=1.003632 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=1 records=10 total_expected_recv=327756 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76672 peak_demand=32990 required_capacity=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=1 records=10 total_expected_recv=327735 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=77056 peak_demand=33069 required_capacity=1.009186 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=1 records=10 total_expected_recv=326598 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=77568 peak_demand=33028 required_capacity=1.007935 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=1 records=10 total_expected_recv=327990 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76544 peak_demand=33011 required_capacity=1.007416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=1 records=10 total_expected_recv=327608 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=77184 peak_demand=33024 required_capacity=1.007812 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=1 records=10 total_expected_recv=328076 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76288 peak_demand=32942 required_capacity=1.005310 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=1 records=10 total_expected_recv=327904 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76160 peak_demand=33087 required_capacity=1.009735 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=1 records=10 total_expected_recv=327773 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76800 peak_demand=33017 required_capacity=1.007599 + [2026-08-26 15:53:11] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 319.7 | throughput per GPU (TFLOP/s/GPU): 386.3 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.097681E+01 | loss scale: 1.0 | grad norm: 30.506 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 33664 | min safe capacity: 1.0273 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=0 calls=1 budget=40960 expected_recv=32746 expected_overflow=0 exact_recv_per_rank=[32610, 32701, 32789, 32689, 32746, 33096, 32500, 33013] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=0 calls=1 budget=40960 expected_recv=32701 expected_overflow=0 exact_recv_per_rank=[32610, 32701, 32789, 32689, 32746, 33096, 32500, 33013] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=0 calls=1 budget=40960 expected_recv=33013 expected_overflow=0 exact_recv_per_rank=[32610, 32701, 32789, 32689, 32746, 33096, 32500, 33013] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=0 calls=1 budget=40960 expected_recv=32789 expected_overflow=0 exact_recv_per_rank=[32610, 32701, 32789, 32689, 32746, 33096, 32500, 33013] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=0 calls=1 budget=40960 expected_recv=32500 expected_overflow=0 exact_recv_per_rank=[32610, 32701, 32789, 32689, 32746, 33096, 32500, 33013] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=0 calls=1 budget=40960 expected_recv=32689 expected_overflow=0 exact_recv_per_rank=[32610, 32701, 32789, 32689, 32746, 33096, 32500, 33013] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32610, 32701, 32789, 32689, 32746, 33096, 32500, 33013] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=0 calls=1 budget=40960 expected_recv=32746 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=0 calls=1 budget=40960 expected_recv=32701 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=0 calls=1 budget=40960 expected_recv=32610 expected_overflow=0 exact_recv_per_rank=[32610, 32701, 32789, 32689, 32746, 33096, 32500, 33013] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=0 calls=1 budget=40960 expected_recv=33013 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=0 calls=1 budget=40960 expected_recv=32789 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=0 calls=1 budget=40960 expected_recv=32500 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=0 calls=1 budget=40960 expected_recv=32689 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=0 calls=1 budget=40960 expected_recv=32610 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=1 calls=1 budget=40960 expected_recv=32642 expected_overflow=0 exact_recv_per_rank=[32826, 32642, 32496, 32979, 32620, 32968, 32803, 32810] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=1 calls=1 budget=40960 expected_recv=32803 expected_overflow=0 exact_recv_per_rank=[32826, 32642, 32496, 32979, 32620, 32968, 32803, 32810] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=1 calls=1 budget=40960 expected_recv=32620 expected_overflow=0 exact_recv_per_rank=[32826, 32642, 32496, 32979, 32620, 32968, 32803, 32810] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=1 calls=1 budget=40960 expected_recv=32810 expected_overflow=0 exact_recv_per_rank=[32826, 32642, 32496, 32979, 32620, 32968, 32803, 32810] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=1 calls=1 budget=40960 expected_recv=32496 expected_overflow=0 exact_recv_per_rank=[32826, 32642, 32496, 32979, 32620, 32968, 32803, 32810] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=1 calls=1 budget=40960 expected_recv=32642 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=1 calls=1 budget=40960 expected_recv=32803 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=1 calls=1 budget=40960 expected_recv=32620 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=1 calls=1 budget=40960 expected_recv=32810 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=1 calls=1 budget=40960 expected_recv=32968 expected_overflow=0 exact_recv_per_rank=[32826, 32642, 32496, 32979, 32620, 32968, 32803, 32810] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=1 calls=1 budget=40960 expected_recv=32826 expected_overflow=0 exact_recv_per_rank=[32826, 32642, 32496, 32979, 32620, 32968, 32803, 32810] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=1 calls=1 budget=40960 expected_recv=32979 expected_overflow=0 exact_recv_per_rank=[32826, 32642, 32496, 32979, 32620, 32968, 32803, 32810] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=1 calls=1 budget=40960 expected_recv=32496 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=1 calls=1 budget=40960 expected_recv=32826 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=1 calls=1 budget=40960 expected_recv=32968 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=1 calls=1 budget=40960 expected_recv=32979 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=2 calls=1 budget=40960 expected_recv=32706 expected_overflow=0 exact_recv_per_rank=[32687, 32770, 33031, 32863, 32696, 32729, 32706, 32662] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=2 calls=1 budget=40960 expected_recv=32662 expected_overflow=0 exact_recv_per_rank=[32687, 32770, 33031, 32863, 32696, 32729, 32706, 32662] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=2 calls=1 budget=40960 expected_recv=32696 expected_overflow=0 exact_recv_per_rank=[32687, 32770, 33031, 32863, 32696, 32729, 32706, 32662] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=2 calls=1 budget=40960 expected_recv=32706 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=2 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[32687, 32770, 33031, 32863, 32696, 32729, 32706, 32662] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=2 calls=1 budget=40960 expected_recv=32696 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=2 calls=1 budget=40960 expected_recv=32662 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=2 calls=1 budget=40960 expected_recv=32687 expected_overflow=0 exact_recv_per_rank=[32687, 32770, 33031, 32863, 32696, 32729, 32706, 32662] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=2 calls=1 budget=40960 expected_recv=32863 expected_overflow=0 exact_recv_per_rank=[32687, 32770, 33031, 32863, 32696, 32729, 32706, 32662] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=2 calls=1 budget=40960 expected_recv=32729 expected_overflow=0 exact_recv_per_rank=[32687, 32770, 33031, 32863, 32696, 32729, 32706, 32662] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=2 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=2 calls=1 budget=40960 expected_recv=33031 expected_overflow=0 exact_recv_per_rank=[32687, 32770, 33031, 32863, 32696, 32729, 32706, 32662] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=2 calls=1 budget=40960 expected_recv=32687 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=2 calls=1 budget=40960 expected_recv=32863 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=2 calls=1 budget=40960 expected_recv=32729 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=2 calls=1 budget=40960 expected_recv=33031 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=3 calls=1 budget=40960 expected_recv=32771 expected_overflow=0 exact_recv_per_rank=[32909, 32776, 32707, 32644, 32768, 32982, 32771, 32587] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=3 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32909, 32776, 32707, 32644, 32768, 32982, 32771, 32587] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=3 calls=1 budget=40960 expected_recv=32587 expected_overflow=0 exact_recv_per_rank=[32909, 32776, 32707, 32644, 32768, 32982, 32771, 32587] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=3 calls=1 budget=40960 expected_recv=32776 expected_overflow=0 exact_recv_per_rank=[32909, 32776, 32707, 32644, 32768, 32982, 32771, 32587] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=3 calls=1 budget=40960 expected_recv=32982 expected_overflow=0 exact_recv_per_rank=[32909, 32776, 32707, 32644, 32768, 32982, 32771, 32587] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=3 calls=1 budget=40960 expected_recv=32587 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=3 calls=1 budget=40960 expected_recv=32644 expected_overflow=0 exact_recv_per_rank=[32909, 32776, 32707, 32644, 32768, 32982, 32771, 32587] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=3 calls=1 budget=40960 expected_recv=32771 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=3 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=3 calls=1 budget=40960 expected_recv=32909 expected_overflow=0 exact_recv_per_rank=[32909, 32776, 32707, 32644, 32768, 32982, 32771, 32587] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=3 calls=1 budget=40960 expected_recv=32707 expected_overflow=0 exact_recv_per_rank=[32909, 32776, 32707, 32644, 32768, 32982, 32771, 32587] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=3 calls=1 budget=40960 expected_recv=32776 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=3 calls=1 budget=40960 expected_recv=32982 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=3 calls=1 budget=40960 expected_recv=32644 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=3 calls=1 budget=40960 expected_recv=32909 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=3 calls=1 budget=40960 expected_recv=32707 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=4 calls=1 budget=40960 expected_recv=32862 expected_overflow=0 exact_recv_per_rank=[32689, 32299, 32901, 32912, 32863, 32694, 32924, 32862] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=4 calls=1 budget=40960 expected_recv=32924 expected_overflow=0 exact_recv_per_rank=[32689, 32299, 32901, 32912, 32863, 32694, 32924, 32862] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=4 calls=1 budget=40960 expected_recv=32863 expected_overflow=0 exact_recv_per_rank=[32689, 32299, 32901, 32912, 32863, 32694, 32924, 32862] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=4 calls=1 budget=40960 expected_recv=32862 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=4 calls=1 budget=40960 expected_recv=32924 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=4 calls=1 budget=40960 expected_recv=32694 expected_overflow=0 exact_recv_per_rank=[32689, 32299, 32901, 32912, 32863, 32694, 32924, 32862] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=4 calls=1 budget=40960 expected_recv=32863 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=4 calls=1 budget=40960 expected_recv=32299 expected_overflow=0 exact_recv_per_rank=[32689, 32299, 32901, 32912, 32863, 32694, 32924, 32862] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=4 calls=1 budget=40960 expected_recv=32912 expected_overflow=0 exact_recv_per_rank=[32689, 32299, 32901, 32912, 32863, 32694, 32924, 32862] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=4 calls=1 budget=40960 expected_recv=32689 expected_overflow=0 exact_recv_per_rank=[32689, 32299, 32901, 32912, 32863, 32694, 32924, 32862] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=4 calls=1 budget=40960 expected_recv=32901 expected_overflow=0 exact_recv_per_rank=[32689, 32299, 32901, 32912, 32863, 32694, 32924, 32862] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=4 calls=1 budget=40960 expected_recv=32912 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=4 calls=1 budget=40960 expected_recv=32689 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=4 calls=1 budget=40960 expected_recv=32694 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=4 calls=1 budget=40960 expected_recv=32299 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=4 calls=1 budget=40960 expected_recv=32901 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=5 calls=1 budget=40960 expected_recv=32899 expected_overflow=0 exact_recv_per_rank=[32923, 32612, 32782, 32709, 32739, 32705, 32775, 32899] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=5 calls=1 budget=40960 expected_recv=32775 expected_overflow=0 exact_recv_per_rank=[32923, 32612, 32782, 32709, 32739, 32705, 32775, 32899] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=5 calls=1 budget=40960 expected_recv=32739 expected_overflow=0 exact_recv_per_rank=[32923, 32612, 32782, 32709, 32739, 32705, 32775, 32899] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=5 calls=1 budget=40960 expected_recv=32899 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=5 calls=1 budget=40960 expected_recv=32775 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=5 calls=1 budget=40960 expected_recv=32739 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=5 calls=1 budget=40960 expected_recv=32709 expected_overflow=0 exact_recv_per_rank=[32923, 32612, 32782, 32709, 32739, 32705, 32775, 32899] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=5 calls=1 budget=40960 expected_recv=32923 expected_overflow=0 exact_recv_per_rank=[32923, 32612, 32782, 32709, 32739, 32705, 32775, 32899] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=5 calls=1 budget=40960 expected_recv=32705 expected_overflow=0 exact_recv_per_rank=[32923, 32612, 32782, 32709, 32739, 32705, 32775, 32899] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=5 calls=1 budget=40960 expected_recv=32612 expected_overflow=0 exact_recv_per_rank=[32923, 32612, 32782, 32709, 32739, 32705, 32775, 32899] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=5 calls=1 budget=40960 expected_recv=32782 expected_overflow=0 exact_recv_per_rank=[32923, 32612, 32782, 32709, 32739, 32705, 32775, 32899] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=5 calls=1 budget=40960 expected_recv=32709 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=5 calls=1 budget=40960 expected_recv=32923 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=5 calls=1 budget=40960 expected_recv=32705 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=5 calls=1 budget=40960 expected_recv=32612 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=5 calls=1 budget=40960 expected_recv=32782 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=6 calls=1 budget=40960 expected_recv=33036 expected_overflow=0 exact_recv_per_rank=[32881, 32764, 32567, 32848, 32797, 32698, 32553, 33036] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=6 calls=1 budget=40960 expected_recv=32553 expected_overflow=0 exact_recv_per_rank=[32881, 32764, 32567, 32848, 32797, 32698, 32553, 33036] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=6 calls=1 budget=40960 expected_recv=33036 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=6 calls=1 budget=40960 expected_recv=32797 expected_overflow=0 exact_recv_per_rank=[32881, 32764, 32567, 32848, 32797, 32698, 32553, 33036] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=6 calls=1 budget=40960 expected_recv=32553 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=6 calls=1 budget=40960 expected_recv=32797 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=6 calls=1 budget=40960 expected_recv=32881 expected_overflow=0 exact_recv_per_rank=[32881, 32764, 32567, 32848, 32797, 32698, 32553, 33036] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=6 calls=1 budget=40960 expected_recv=32848 expected_overflow=0 exact_recv_per_rank=[32881, 32764, 32567, 32848, 32797, 32698, 32553, 33036] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=6 calls=1 budget=40960 expected_recv=32698 expected_overflow=0 exact_recv_per_rank=[32881, 32764, 32567, 32848, 32797, 32698, 32553, 33036] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=6 calls=1 budget=40960 expected_recv=32764 expected_overflow=0 exact_recv_per_rank=[32881, 32764, 32567, 32848, 32797, 32698, 32553, 33036] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=6 calls=1 budget=40960 expected_recv=32567 expected_overflow=0 exact_recv_per_rank=[32881, 32764, 32567, 32848, 32797, 32698, 32553, 33036] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=6 calls=1 budget=40960 expected_recv=32881 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=6 calls=1 budget=40960 expected_recv=32848 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=6 calls=1 budget=40960 expected_recv=32698 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=6 calls=1 budget=40960 expected_recv=32764 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=6 calls=1 budget=40960 expected_recv=32567 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=7 calls=1 budget=40960 expected_recv=32748 expected_overflow=0 exact_recv_per_rank=[32699, 32934, 32491, 32805, 32878, 32883, 32706, 32748] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=7 calls=1 budget=40960 expected_recv=32706 expected_overflow=0 exact_recv_per_rank=[32699, 32934, 32491, 32805, 32878, 32883, 32706, 32748] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=7 calls=1 budget=40960 expected_recv=32878 expected_overflow=0 exact_recv_per_rank=[32699, 32934, 32491, 32805, 32878, 32883, 32706, 32748] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=7 calls=1 budget=40960 expected_recv=32748 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=7 calls=1 budget=40960 expected_recv=32706 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=7 calls=1 budget=40960 expected_recv=32878 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=7 calls=1 budget=40960 expected_recv=32883 expected_overflow=0 exact_recv_per_rank=[32699, 32934, 32491, 32805, 32878, 32883, 32706, 32748] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=7 calls=1 budget=40960 expected_recv=32699 expected_overflow=0 exact_recv_per_rank=[32699, 32934, 32491, 32805, 32878, 32883, 32706, 32748] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=7 calls=1 budget=40960 expected_recv=32805 expected_overflow=0 exact_recv_per_rank=[32699, 32934, 32491, 32805, 32878, 32883, 32706, 32748] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=7 calls=1 budget=40960 expected_recv=32491 expected_overflow=0 exact_recv_per_rank=[32699, 32934, 32491, 32805, 32878, 32883, 32706, 32748] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=7 calls=1 budget=40960 expected_recv=32934 expected_overflow=0 exact_recv_per_rank=[32699, 32934, 32491, 32805, 32878, 32883, 32706, 32748] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=7 calls=1 budget=40960 expected_recv=32883 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=7 calls=1 budget=40960 expected_recv=32699 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=7 calls=1 budget=40960 expected_recv=32805 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=7 calls=1 budget=40960 expected_recv=32491 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=7 calls=1 budget=40960 expected_recv=32934 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=8 calls=1 budget=40960 expected_recv=32979 expected_overflow=0 exact_recv_per_rank=[32531, 32519, 32502, 32908, 33096, 32812, 32797, 32979] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=8 calls=1 budget=40960 expected_recv=32979 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=8 calls=1 budget=40960 expected_recv=32797 expected_overflow=0 exact_recv_per_rank=[32531, 32519, 32502, 32908, 33096, 32812, 32797, 32979] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=8 calls=1 budget=40960 expected_recv=32797 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=8 calls=1 budget=40960 expected_recv=32812 expected_overflow=0 exact_recv_per_rank=[32531, 32519, 32502, 32908, 33096, 32812, 32797, 32979] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=8 calls=1 budget=40960 expected_recv=32531 expected_overflow=0 exact_recv_per_rank=[32531, 32519, 32502, 32908, 33096, 32812, 32797, 32979] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=8 calls=1 budget=40960 expected_recv=32908 expected_overflow=0 exact_recv_per_rank=[32531, 32519, 32502, 32908, 33096, 32812, 32797, 32979] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=8 calls=1 budget=40960 expected_recv=32502 expected_overflow=0 exact_recv_per_rank=[32531, 32519, 32502, 32908, 33096, 32812, 32797, 32979] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=8 calls=1 budget=40960 expected_recv=32812 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32531, 32519, 32502, 32908, 33096, 32812, 32797, 32979] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=8 calls=1 budget=40960 expected_recv=32519 expected_overflow=0 exact_recv_per_rank=[32531, 32519, 32502, 32908, 33096, 32812, 32797, 32979] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=8 calls=1 budget=40960 expected_recv=32531 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=8 calls=1 budget=40960 expected_recv=32908 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=8 calls=1 budget=40960 expected_recv=32502 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=8 calls=1 budget=40960 expected_recv=32519 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=9 calls=1 budget=40960 expected_recv=32993 expected_overflow=0 exact_recv_per_rank=[32615, 32608, 32627, 32757, 32817, 32842, 32885, 32993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=9 calls=1 budget=40960 expected_recv=32885 expected_overflow=0 exact_recv_per_rank=[32615, 32608, 32627, 32757, 32817, 32842, 32885, 32993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=9 calls=1 budget=40960 expected_recv=32993 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=9 calls=1 budget=40960 expected_recv=32885 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=9 calls=1 budget=40960 expected_recv=32842 expected_overflow=0 exact_recv_per_rank=[32615, 32608, 32627, 32757, 32817, 32842, 32885, 32993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=9 calls=1 budget=40960 expected_recv=32842 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=9 calls=1 budget=40960 expected_recv=32615 expected_overflow=0 exact_recv_per_rank=[32615, 32608, 32627, 32757, 32817, 32842, 32885, 32993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=9 calls=1 budget=40960 expected_recv=32757 expected_overflow=0 exact_recv_per_rank=[32615, 32608, 32627, 32757, 32817, 32842, 32885, 32993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=9 calls=1 budget=40960 expected_recv=32627 expected_overflow=0 exact_recv_per_rank=[32615, 32608, 32627, 32757, 32817, 32842, 32885, 32993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=9 calls=1 budget=40960 expected_recv=32817 expected_overflow=0 exact_recv_per_rank=[32615, 32608, 32627, 32757, 32817, 32842, 32885, 32993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=9 calls=1 budget=40960 expected_recv=32608 expected_overflow=0 exact_recv_per_rank=[32615, 32608, 32627, 32757, 32817, 32842, 32885, 32993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=9 calls=1 budget=40960 expected_recv=32615 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=9 calls=1 budget=40960 expected_recv=32757 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=9 calls=1 budget=40960 expected_recv=32627 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=9 calls=1 budget=40960 expected_recv=32817 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=1 tokens_per_ep_rank=[32610, 32701, 32789, 32689, 32746, 33121, 32475, 33013] avg=32768.0000 max=33121 max_rank=5 min=32475 min_rank=6 imbalance_ratio=1.010773 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=9 calls=1 budget=40960 expected_recv=32608 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=1 tokens_per_ep_rank=[32610, 32701, 32789, 32689, 32746, 33121, 32475, 33013] avg=32768.0000 max=33121 max_rank=5 min=32475 min_rank=6 imbalance_ratio=1.010773 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=1 tokens_per_ep_rank=[32610, 32701, 32789, 32689, 32746, 33096, 32500, 33013] avg=32768.0000 max=33096 max_rank=5 min=32500 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=25 max_load_reduction_pct=0.000755 moved_to_replicas=25 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=1 tokens_per_ep_rank=[32610, 32701, 32789, 32689, 32746, 33121, 32475, 33013] avg=32768.0000 max=33121 max_rank=5 min=32475 min_rank=6 imbalance_ratio=1.010773 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=1 tokens_per_ep_rank=[32610, 32701, 32789, 32689, 32746, 33096, 32500, 33013] avg=32768.0000 max=33096 max_rank=5 min=32500 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=25 max_load_reduction_pct=0.000755 moved_to_replicas=25 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=1 tokens_per_ep_rank=[32610, 32701, 32789, 32689, 32746, 33121, 32475, 33013] avg=32768.0000 max=33121 max_rank=5 min=32475 min_rank=6 imbalance_ratio=1.010773 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=1 tokens_per_ep_rank=[32610, 32701, 32789, 32689, 32746, 33121, 32475, 33013] avg=32768.0000 max=33121 max_rank=5 min=32475 min_rank=6 imbalance_ratio=1.010773 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=1 tokens_per_ep_rank=[32610, 32701, 32789, 32689, 32746, 33121, 32475, 33013] avg=32768.0000 max=33121 max_rank=5 min=32475 min_rank=6 imbalance_ratio=1.010773 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=1 tokens_per_ep_rank=[32610, 32701, 32789, 32689, 32746, 33121, 32475, 33013] avg=32768.0000 max=33121 max_rank=5 min=32475 min_rank=6 imbalance_ratio=1.010773 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=1 tokens_per_ep_rank=[32610, 32701, 32789, 32689, 32746, 33121, 32475, 33013] avg=32768.0000 max=33121 max_rank=5 min=32475 min_rank=6 imbalance_ratio=1.010773 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=1 tokens_per_ep_rank=[32610, 32701, 32789, 32689, 32746, 33096, 32500, 33013] avg=32768.0000 max=33096 max_rank=5 min=32500 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=25 max_load_reduction_pct=0.000755 moved_to_replicas=25 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=2 tokens_per_ep_rank=[32826, 32642, 32496, 32979, 32620, 32968, 32803, 32810] avg=32768.0000 max=32979 max_rank=3 min=32496 min_rank=2 imbalance_ratio=1.006439 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=1 tokens_per_ep_rank=[32610, 32701, 32789, 32689, 32746, 33096, 32500, 33013] avg=32768.0000 max=33096 max_rank=5 min=32500 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=25 max_load_reduction_pct=0.000755 moved_to_replicas=25 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=2 tokens_per_ep_rank=[32826, 32642, 32496, 32979, 32620, 32968, 32803, 32810] avg=32768.0000 max=32979 max_rank=3 min=32496 min_rank=2 imbalance_ratio=1.006439 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=1 tokens_per_ep_rank=[32610, 32701, 32789, 32689, 32746, 33096, 32500, 33013] avg=32768.0000 max=33096 max_rank=5 min=32500 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=25 max_load_reduction_pct=0.000755 moved_to_replicas=25 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=1 tokens_per_ep_rank=[32610, 32701, 32789, 32689, 32746, 33096, 32500, 33013] avg=32768.0000 max=33096 max_rank=5 min=32500 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=25 max_load_reduction_pct=0.000755 moved_to_replicas=25 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=1 tokens_per_ep_rank=[32610, 32701, 32789, 32689, 32746, 33096, 32500, 33013] avg=32768.0000 max=33096 max_rank=5 min=32500 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=25 max_load_reduction_pct=0.000755 moved_to_replicas=25 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=1 tokens_per_ep_rank=[32610, 32701, 32789, 32689, 32746, 33096, 32500, 33013] avg=32768.0000 max=33096 max_rank=5 min=32500 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=25 max_load_reduction_pct=0.000755 moved_to_replicas=25 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=2 tokens_per_ep_rank=[32826, 32642, 32496, 32979, 32620, 32968, 32803, 32810] avg=32768.0000 max=32979 max_rank=3 min=32496 min_rank=2 imbalance_ratio=1.006439 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=2 tokens_per_ep_rank=[32826, 32642, 32496, 32979, 32620, 32968, 32803, 32810] avg=32768.0000 max=32979 max_rank=3 min=32496 min_rank=2 imbalance_ratio=1.006439 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=2 tokens_per_ep_rank=[32826, 32642, 32496, 32979, 32620, 32968, 32803, 32810] avg=32768.0000 max=32979 max_rank=3 min=32496 min_rank=2 imbalance_ratio=1.006439 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=2 tokens_per_ep_rank=[32826, 32642, 32496, 32979, 32620, 32968, 32803, 32810] avg=32768.0000 max=32979 max_rank=3 min=32496 min_rank=2 imbalance_ratio=1.006439 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=2 tokens_per_ep_rank=[32826, 32642, 32496, 32979, 32620, 32968, 32803, 32810] avg=32768.0000 max=32979 max_rank=3 min=32496 min_rank=2 imbalance_ratio=1.006439 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=2 tokens_per_ep_rank=[32826, 32642, 32496, 32979, 32620, 32968, 32803, 32810] avg=32768.0000 max=32979 max_rank=3 min=32496 min_rank=2 imbalance_ratio=1.006439 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=2 tokens_per_ep_rank=[32826, 32642, 32496, 32979, 32620, 32968, 32803, 32810] avg=32768.0000 max=32979 max_rank=3 min=32496 min_rank=2 imbalance_ratio=1.006439 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=2 tokens_per_ep_rank=[32826, 32642, 32496, 32979, 32620, 32968, 32803, 32810] avg=32768.0000 max=32979 max_rank=3 min=32496 min_rank=2 imbalance_ratio=1.006439 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=3 tokens_per_ep_rank=[32687, 32770, 33031, 32863, 32696, 32729, 32706, 32662] avg=32768.0000 max=33031 max_rank=2 min=32662 min_rank=7 imbalance_ratio=1.008026 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=2 tokens_per_ep_rank=[32826, 32642, 32496, 32979, 32620, 32968, 32803, 32810] avg=32768.0000 max=32979 max_rank=3 min=32496 min_rank=2 imbalance_ratio=1.006439 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=2 tokens_per_ep_rank=[32826, 32642, 32496, 32979, 32620, 32968, 32803, 32810] avg=32768.0000 max=32979 max_rank=3 min=32496 min_rank=2 imbalance_ratio=1.006439 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=3 tokens_per_ep_rank=[32687, 32770, 33031, 32863, 32696, 32729, 32706, 32662] avg=32768.0000 max=33031 max_rank=2 min=32662 min_rank=7 imbalance_ratio=1.008026 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=2 tokens_per_ep_rank=[32826, 32642, 32496, 32979, 32620, 32968, 32803, 32810] avg=32768.0000 max=32979 max_rank=3 min=32496 min_rank=2 imbalance_ratio=1.006439 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=2 tokens_per_ep_rank=[32826, 32642, 32496, 32979, 32620, 32968, 32803, 32810] avg=32768.0000 max=32979 max_rank=3 min=32496 min_rank=2 imbalance_ratio=1.006439 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=3 tokens_per_ep_rank=[32687, 32770, 33031, 32863, 32696, 32729, 32706, 32662] avg=32768.0000 max=33031 max_rank=2 min=32662 min_rank=7 imbalance_ratio=1.008026 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=2 tokens_per_ep_rank=[32826, 32642, 32496, 32979, 32620, 32968, 32803, 32810] avg=32768.0000 max=32979 max_rank=3 min=32496 min_rank=2 imbalance_ratio=1.006439 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=2 tokens_per_ep_rank=[32826, 32642, 32496, 32979, 32620, 32968, 32803, 32810] avg=32768.0000 max=32979 max_rank=3 min=32496 min_rank=2 imbalance_ratio=1.006439 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=3 tokens_per_ep_rank=[32687, 32770, 33031, 32863, 32696, 32729, 32706, 32662] avg=32768.0000 max=33031 max_rank=2 min=32662 min_rank=7 imbalance_ratio=1.008026 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=3 tokens_per_ep_rank=[32687, 32770, 33031, 32863, 32696, 32729, 32706, 32662] avg=32768.0000 max=33031 max_rank=2 min=32662 min_rank=7 imbalance_ratio=1.008026 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=3 tokens_per_ep_rank=[32687, 32770, 33031, 32863, 32696, 32729, 32706, 32662] avg=32768.0000 max=33031 max_rank=2 min=32662 min_rank=7 imbalance_ratio=1.008026 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=3 tokens_per_ep_rank=[32687, 32770, 33031, 32863, 32696, 32729, 32706, 32662] avg=32768.0000 max=33031 max_rank=2 min=32662 min_rank=7 imbalance_ratio=1.008026 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=4 tokens_per_ep_rank=[32909, 32776, 32707, 32644, 32768, 32982, 32771, 32587] avg=32768.0000 max=32982 max_rank=5 min=32587 min_rank=7 imbalance_ratio=1.006531 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=3 tokens_per_ep_rank=[32687, 32770, 33031, 32863, 32696, 32729, 32706, 32662] avg=32768.0000 max=33031 max_rank=2 min=32662 min_rank=7 imbalance_ratio=1.008026 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=3 tokens_per_ep_rank=[32687, 32770, 33031, 32863, 32696, 32729, 32706, 32662] avg=32768.0000 max=33031 max_rank=2 min=32662 min_rank=7 imbalance_ratio=1.008026 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=3 tokens_per_ep_rank=[32687, 32770, 33031, 32863, 32696, 32729, 32706, 32662] avg=32768.0000 max=33031 max_rank=2 min=32662 min_rank=7 imbalance_ratio=1.008026 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=3 tokens_per_ep_rank=[32687, 32770, 33031, 32863, 32696, 32729, 32706, 32662] avg=32768.0000 max=33031 max_rank=2 min=32662 min_rank=7 imbalance_ratio=1.008026 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=4 tokens_per_ep_rank=[32909, 32776, 32707, 32644, 32768, 32982, 32771, 32587] avg=32768.0000 max=32982 max_rank=5 min=32587 min_rank=7 imbalance_ratio=1.006531 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=3 tokens_per_ep_rank=[32687, 32770, 33031, 32863, 32696, 32729, 32706, 32662] avg=32768.0000 max=33031 max_rank=2 min=32662 min_rank=7 imbalance_ratio=1.008026 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=3 tokens_per_ep_rank=[32687, 32770, 33031, 32863, 32696, 32729, 32706, 32662] avg=32768.0000 max=33031 max_rank=2 min=32662 min_rank=7 imbalance_ratio=1.008026 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=4 tokens_per_ep_rank=[32909, 32776, 32707, 32644, 32768, 32982, 32771, 32587] avg=32768.0000 max=32982 max_rank=5 min=32587 min_rank=7 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=3 tokens_per_ep_rank=[32687, 32770, 33031, 32863, 32696, 32729, 32706, 32662] avg=32768.0000 max=33031 max_rank=2 min=32662 min_rank=7 imbalance_ratio=1.008026 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=3 tokens_per_ep_rank=[32687, 32770, 33031, 32863, 32696, 32729, 32706, 32662] avg=32768.0000 max=33031 max_rank=2 min=32662 min_rank=7 imbalance_ratio=1.008026 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=3 tokens_per_ep_rank=[32687, 32770, 33031, 32863, 32696, 32729, 32706, 32662] avg=32768.0000 max=33031 max_rank=2 min=32662 min_rank=7 imbalance_ratio=1.008026 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=4 tokens_per_ep_rank=[32909, 32776, 32707, 32644, 32768, 32982, 32771, 32587] avg=32768.0000 max=32982 max_rank=5 min=32587 min_rank=7 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=4 tokens_per_ep_rank=[32909, 32776, 32707, 32644, 32768, 32982, 32771, 32587] avg=32768.0000 max=32982 max_rank=5 min=32587 min_rank=7 imbalance_ratio=1.006531 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=4 tokens_per_ep_rank=[32909, 32776, 32707, 32644, 32768, 32982, 32771, 32587] avg=32768.0000 max=32982 max_rank=5 min=32587 min_rank=7 imbalance_ratio=1.006531 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=4 tokens_per_ep_rank=[32909, 32776, 32707, 32644, 32768, 32982, 32771, 32587] avg=32768.0000 max=32982 max_rank=5 min=32587 min_rank=7 imbalance_ratio=1.006531 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=5 tokens_per_ep_rank=[32689, 32299, 32901, 32912, 32863, 32694, 32924, 32862] avg=32768.0000 max=32924 max_rank=6 min=32299 min_rank=1 imbalance_ratio=1.004761 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=4 tokens_per_ep_rank=[32909, 32776, 32707, 32644, 32768, 32982, 32771, 32587] avg=32768.0000 max=32982 max_rank=5 min=32587 min_rank=7 imbalance_ratio=1.006531 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=4 tokens_per_ep_rank=[32909, 32776, 32707, 32644, 32768, 32982, 32771, 32587] avg=32768.0000 max=32982 max_rank=5 min=32587 min_rank=7 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=4 tokens_per_ep_rank=[32909, 32776, 32707, 32644, 32768, 32982, 32771, 32587] avg=32768.0000 max=32982 max_rank=5 min=32587 min_rank=7 imbalance_ratio=1.006531 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=4 tokens_per_ep_rank=[32909, 32776, 32707, 32644, 32768, 32982, 32771, 32587] avg=32768.0000 max=32982 max_rank=5 min=32587 min_rank=7 imbalance_ratio=1.006531 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=5 tokens_per_ep_rank=[32689, 32299, 32901, 32912, 32863, 32694, 32924, 32862] avg=32768.0000 max=32924 max_rank=6 min=32299 min_rank=1 imbalance_ratio=1.004761 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=4 tokens_per_ep_rank=[32909, 32776, 32707, 32644, 32768, 32982, 32771, 32587] avg=32768.0000 max=32982 max_rank=5 min=32587 min_rank=7 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=4 tokens_per_ep_rank=[32909, 32776, 32707, 32644, 32768, 32982, 32771, 32587] avg=32768.0000 max=32982 max_rank=5 min=32587 min_rank=7 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=5 tokens_per_ep_rank=[32689, 32299, 32901, 32912, 32863, 32694, 32924, 32862] avg=32768.0000 max=32924 max_rank=6 min=32299 min_rank=1 imbalance_ratio=1.004761 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=4 tokens_per_ep_rank=[32909, 32776, 32707, 32644, 32768, 32982, 32771, 32587] avg=32768.0000 max=32982 max_rank=5 min=32587 min_rank=7 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=4 tokens_per_ep_rank=[32909, 32776, 32707, 32644, 32768, 32982, 32771, 32587] avg=32768.0000 max=32982 max_rank=5 min=32587 min_rank=7 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=4 tokens_per_ep_rank=[32909, 32776, 32707, 32644, 32768, 32982, 32771, 32587] avg=32768.0000 max=32982 max_rank=5 min=32587 min_rank=7 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=5 tokens_per_ep_rank=[32689, 32299, 32901, 32912, 32863, 32694, 32924, 32862] avg=32768.0000 max=32924 max_rank=6 min=32299 min_rank=1 imbalance_ratio=1.004761 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=5 tokens_per_ep_rank=[32689, 32299, 32901, 32912, 32863, 32694, 32924, 32862] avg=32768.0000 max=32924 max_rank=6 min=32299 min_rank=1 imbalance_ratio=1.004761 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=5 tokens_per_ep_rank=[32689, 32299, 32901, 32912, 32863, 32694, 32924, 32862] avg=32768.0000 max=32924 max_rank=6 min=32299 min_rank=1 imbalance_ratio=1.004761 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=5 tokens_per_ep_rank=[32689, 32299, 32901, 32912, 32863, 32694, 32924, 32862] avg=32768.0000 max=32924 max_rank=6 min=32299 min_rank=1 imbalance_ratio=1.004761 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=6 tokens_per_ep_rank=[32923, 32612, 32782, 32709, 32739, 32705, 32775, 32899] avg=32768.0000 max=32923 max_rank=0 min=32612 min_rank=1 imbalance_ratio=1.004730 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=5 tokens_per_ep_rank=[32689, 32299, 32901, 32912, 32863, 32694, 32924, 32862] avg=32768.0000 max=32924 max_rank=6 min=32299 min_rank=1 imbalance_ratio=1.004761 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=5 tokens_per_ep_rank=[32689, 32299, 32901, 32912, 32863, 32694, 32924, 32862] avg=32768.0000 max=32924 max_rank=6 min=32299 min_rank=1 imbalance_ratio=1.004761 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=5 tokens_per_ep_rank=[32689, 32299, 32901, 32912, 32863, 32694, 32924, 32862] avg=32768.0000 max=32924 max_rank=6 min=32299 min_rank=1 imbalance_ratio=1.004761 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=5 tokens_per_ep_rank=[32689, 32299, 32901, 32912, 32863, 32694, 32924, 32862] avg=32768.0000 max=32924 max_rank=6 min=32299 min_rank=1 imbalance_ratio=1.004761 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=6 tokens_per_ep_rank=[32923, 32612, 32782, 32709, 32739, 32705, 32775, 32899] avg=32768.0000 max=32923 max_rank=0 min=32612 min_rank=1 imbalance_ratio=1.004730 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=5 tokens_per_ep_rank=[32689, 32299, 32901, 32912, 32863, 32694, 32924, 32862] avg=32768.0000 max=32924 max_rank=6 min=32299 min_rank=1 imbalance_ratio=1.004761 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=5 tokens_per_ep_rank=[32689, 32299, 32901, 32912, 32863, 32694, 32924, 32862] avg=32768.0000 max=32924 max_rank=6 min=32299 min_rank=1 imbalance_ratio=1.004761 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=6 tokens_per_ep_rank=[32923, 32612, 32782, 32709, 32739, 32705, 32775, 32899] avg=32768.0000 max=32923 max_rank=0 min=32612 min_rank=1 imbalance_ratio=1.004730 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=5 tokens_per_ep_rank=[32689, 32299, 32901, 32912, 32863, 32694, 32924, 32862] avg=32768.0000 max=32924 max_rank=6 min=32299 min_rank=1 imbalance_ratio=1.004761 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=5 tokens_per_ep_rank=[32689, 32299, 32901, 32912, 32863, 32694, 32924, 32862] avg=32768.0000 max=32924 max_rank=6 min=32299 min_rank=1 imbalance_ratio=1.004761 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=5 tokens_per_ep_rank=[32689, 32299, 32901, 32912, 32863, 32694, 32924, 32862] avg=32768.0000 max=32924 max_rank=6 min=32299 min_rank=1 imbalance_ratio=1.004761 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=6 tokens_per_ep_rank=[32923, 32612, 32782, 32709, 32739, 32705, 32775, 32899] avg=32768.0000 max=32923 max_rank=0 min=32612 min_rank=1 imbalance_ratio=1.004730 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=6 tokens_per_ep_rank=[32923, 32612, 32782, 32709, 32739, 32705, 32775, 32899] avg=32768.0000 max=32923 max_rank=0 min=32612 min_rank=1 imbalance_ratio=1.004730 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=6 tokens_per_ep_rank=[32923, 32612, 32782, 32709, 32739, 32705, 32775, 32899] avg=32768.0000 max=32923 max_rank=0 min=32612 min_rank=1 imbalance_ratio=1.004730 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=6 tokens_per_ep_rank=[32923, 32612, 32782, 32709, 32739, 32705, 32775, 32899] avg=32768.0000 max=32923 max_rank=0 min=32612 min_rank=1 imbalance_ratio=1.004730 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=6 tokens_per_ep_rank=[32923, 32612, 32782, 32709, 32739, 32705, 32775, 32899] avg=32768.0000 max=32923 max_rank=0 min=32612 min_rank=1 imbalance_ratio=1.004730 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=6 tokens_per_ep_rank=[32923, 32612, 32782, 32709, 32739, 32705, 32775, 32899] avg=32768.0000 max=32923 max_rank=0 min=32612 min_rank=1 imbalance_ratio=1.004730 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=6 tokens_per_ep_rank=[32923, 32612, 32782, 32709, 32739, 32705, 32775, 32899] avg=32768.0000 max=32923 max_rank=0 min=32612 min_rank=1 imbalance_ratio=1.004730 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=6 tokens_per_ep_rank=[32923, 32612, 32782, 32709, 32739, 32705, 32775, 32899] avg=32768.0000 max=32923 max_rank=0 min=32612 min_rank=1 imbalance_ratio=1.004730 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=7 tokens_per_ep_rank=[32881, 32764, 32567, 32848, 32797, 32698, 32553, 33036] avg=32768.0000 max=33036 max_rank=7 min=32553 min_rank=6 imbalance_ratio=1.008179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=6 tokens_per_ep_rank=[32923, 32612, 32782, 32709, 32739, 32705, 32775, 32899] avg=32768.0000 max=32923 max_rank=0 min=32612 min_rank=1 imbalance_ratio=1.004730 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=7 tokens_per_ep_rank=[32881, 32764, 32567, 32848, 32797, 32698, 32553, 33036] avg=32768.0000 max=33036 max_rank=7 min=32553 min_rank=6 imbalance_ratio=1.008179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=6 tokens_per_ep_rank=[32923, 32612, 32782, 32709, 32739, 32705, 32775, 32899] avg=32768.0000 max=32923 max_rank=0 min=32612 min_rank=1 imbalance_ratio=1.004730 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=6 tokens_per_ep_rank=[32923, 32612, 32782, 32709, 32739, 32705, 32775, 32899] avg=32768.0000 max=32923 max_rank=0 min=32612 min_rank=1 imbalance_ratio=1.004730 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=6 tokens_per_ep_rank=[32923, 32612, 32782, 32709, 32739, 32705, 32775, 32899] avg=32768.0000 max=32923 max_rank=0 min=32612 min_rank=1 imbalance_ratio=1.004730 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=7 tokens_per_ep_rank=[32881, 32764, 32567, 32848, 32797, 32698, 32553, 33036] avg=32768.0000 max=33036 max_rank=7 min=32553 min_rank=6 imbalance_ratio=1.008179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=6 tokens_per_ep_rank=[32923, 32612, 32782, 32709, 32739, 32705, 32775, 32899] avg=32768.0000 max=32923 max_rank=0 min=32612 min_rank=1 imbalance_ratio=1.004730 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=7 tokens_per_ep_rank=[32881, 32764, 32567, 32848, 32797, 32698, 32553, 33036] avg=32768.0000 max=33036 max_rank=7 min=32553 min_rank=6 imbalance_ratio=1.008179 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=7 tokens_per_ep_rank=[32881, 32764, 32567, 32848, 32797, 32698, 32553, 33036] avg=32768.0000 max=33036 max_rank=7 min=32553 min_rank=6 imbalance_ratio=1.008179 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=7 tokens_per_ep_rank=[32881, 32764, 32567, 32848, 32797, 32698, 32553, 33036] avg=32768.0000 max=33036 max_rank=7 min=32553 min_rank=6 imbalance_ratio=1.008179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=7 tokens_per_ep_rank=[32881, 32764, 32567, 32848, 32797, 32698, 32553, 33036] avg=32768.0000 max=33036 max_rank=7 min=32553 min_rank=6 imbalance_ratio=1.008179 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=7 tokens_per_ep_rank=[32881, 32764, 32567, 32848, 32797, 32698, 32553, 33036] avg=32768.0000 max=33036 max_rank=7 min=32553 min_rank=6 imbalance_ratio=1.008179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=7 tokens_per_ep_rank=[32881, 32764, 32567, 32848, 32797, 32698, 32553, 33036] avg=32768.0000 max=33036 max_rank=7 min=32553 min_rank=6 imbalance_ratio=1.008179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=7 tokens_per_ep_rank=[32881, 32764, 32567, 32848, 32797, 32698, 32553, 33036] avg=32768.0000 max=33036 max_rank=7 min=32553 min_rank=6 imbalance_ratio=1.008179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=7 tokens_per_ep_rank=[32881, 32764, 32567, 32848, 32797, 32698, 32553, 33036] avg=32768.0000 max=33036 max_rank=7 min=32553 min_rank=6 imbalance_ratio=1.008179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=7 tokens_per_ep_rank=[32881, 32764, 32567, 32848, 32797, 32698, 32553, 33036] avg=32768.0000 max=33036 max_rank=7 min=32553 min_rank=6 imbalance_ratio=1.008179 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=8 tokens_per_ep_rank=[32699, 32934, 32491, 32805, 32878, 32883, 32706, 32748] avg=32768.0000 max=32934 max_rank=1 min=32491 min_rank=2 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=8 tokens_per_ep_rank=[32699, 32934, 32491, 32805, 32878, 32883, 32706, 32748] avg=32768.0000 max=32934 max_rank=1 min=32491 min_rank=2 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=8 tokens_per_ep_rank=[32699, 32934, 32491, 32805, 32878, 32883, 32706, 32748] avg=32768.0000 max=32934 max_rank=1 min=32491 min_rank=2 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=7 tokens_per_ep_rank=[32881, 32764, 32567, 32848, 32797, 32698, 32553, 33036] avg=32768.0000 max=33036 max_rank=7 min=32553 min_rank=6 imbalance_ratio=1.008179 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=7 tokens_per_ep_rank=[32881, 32764, 32567, 32848, 32797, 32698, 32553, 33036] avg=32768.0000 max=33036 max_rank=7 min=32553 min_rank=6 imbalance_ratio=1.008179 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=7 tokens_per_ep_rank=[32881, 32764, 32567, 32848, 32797, 32698, 32553, 33036] avg=32768.0000 max=33036 max_rank=7 min=32553 min_rank=6 imbalance_ratio=1.008179 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=7 tokens_per_ep_rank=[32881, 32764, 32567, 32848, 32797, 32698, 32553, 33036] avg=32768.0000 max=33036 max_rank=7 min=32553 min_rank=6 imbalance_ratio=1.008179 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=8 tokens_per_ep_rank=[32699, 32934, 32491, 32805, 32878, 32883, 32706, 32748] avg=32768.0000 max=32934 max_rank=1 min=32491 min_rank=2 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=8 tokens_per_ep_rank=[32699, 32934, 32491, 32805, 32878, 32883, 32706, 32748] avg=32768.0000 max=32934 max_rank=1 min=32491 min_rank=2 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=8 tokens_per_ep_rank=[32699, 32934, 32491, 32805, 32878, 32883, 32706, 32748] avg=32768.0000 max=32934 max_rank=1 min=32491 min_rank=2 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=8 tokens_per_ep_rank=[32699, 32934, 32491, 32805, 32878, 32883, 32706, 32748] avg=32768.0000 max=32934 max_rank=1 min=32491 min_rank=2 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=8 tokens_per_ep_rank=[32699, 32934, 32491, 32805, 32878, 32883, 32706, 32748] avg=32768.0000 max=32934 max_rank=1 min=32491 min_rank=2 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=8 tokens_per_ep_rank=[32699, 32934, 32491, 32805, 32878, 32883, 32706, 32748] avg=32768.0000 max=32934 max_rank=1 min=32491 min_rank=2 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=8 tokens_per_ep_rank=[32699, 32934, 32491, 32805, 32878, 32883, 32706, 32748] avg=32768.0000 max=32934 max_rank=1 min=32491 min_rank=2 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=8 tokens_per_ep_rank=[32699, 32934, 32491, 32805, 32878, 32883, 32706, 32748] avg=32768.0000 max=32934 max_rank=1 min=32491 min_rank=2 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=9 tokens_per_ep_rank=[32457, 32519, 32502, 32908, 33170, 32812, 32797, 32979] avg=32768.0000 max=33170 max_rank=4 min=32457 min_rank=0 imbalance_ratio=1.012268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=8 tokens_per_ep_rank=[32699, 32934, 32491, 32805, 32878, 32883, 32706, 32748] avg=32768.0000 max=32934 max_rank=1 min=32491 min_rank=2 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=9 tokens_per_ep_rank=[32457, 32519, 32502, 32908, 33170, 32812, 32797, 32979] avg=32768.0000 max=33170 max_rank=4 min=32457 min_rank=0 imbalance_ratio=1.012268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=9 tokens_per_ep_rank=[32457, 32519, 32502, 32908, 33170, 32812, 32797, 32979] avg=32768.0000 max=33170 max_rank=4 min=32457 min_rank=0 imbalance_ratio=1.012268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=8 tokens_per_ep_rank=[32699, 32934, 32491, 32805, 32878, 32883, 32706, 32748] avg=32768.0000 max=32934 max_rank=1 min=32491 min_rank=2 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=8 tokens_per_ep_rank=[32699, 32934, 32491, 32805, 32878, 32883, 32706, 32748] avg=32768.0000 max=32934 max_rank=1 min=32491 min_rank=2 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=8 tokens_per_ep_rank=[32699, 32934, 32491, 32805, 32878, 32883, 32706, 32748] avg=32768.0000 max=32934 max_rank=1 min=32491 min_rank=2 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=9 tokens_per_ep_rank=[32531, 32519, 32502, 32908, 33096, 32812, 32797, 32979] avg=32768.0000 max=33096 max_rank=4 min=32502 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=74 max_load_reduction_pct=0.002231 moved_to_replicas=74 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=8 tokens_per_ep_rank=[32699, 32934, 32491, 32805, 32878, 32883, 32706, 32748] avg=32768.0000 max=32934 max_rank=1 min=32491 min_rank=2 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=9 tokens_per_ep_rank=[32531, 32519, 32502, 32908, 33096, 32812, 32797, 32979] avg=32768.0000 max=33096 max_rank=4 min=32502 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=74 max_load_reduction_pct=0.002231 moved_to_replicas=74 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=9 tokens_per_ep_rank=[32531, 32519, 32502, 32908, 33096, 32812, 32797, 32979] avg=32768.0000 max=33096 max_rank=4 min=32502 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=74 max_load_reduction_pct=0.002231 moved_to_replicas=74 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=9 tokens_per_ep_rank=[32457, 32519, 32502, 32908, 33170, 32812, 32797, 32979] avg=32768.0000 max=33170 max_rank=4 min=32457 min_rank=0 imbalance_ratio=1.012268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=9 tokens_per_ep_rank=[32457, 32519, 32502, 32908, 33170, 32812, 32797, 32979] avg=32768.0000 max=33170 max_rank=4 min=32457 min_rank=0 imbalance_ratio=1.012268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=9 tokens_per_ep_rank=[32531, 32519, 32502, 32908, 33096, 32812, 32797, 32979] avg=32768.0000 max=33096 max_rank=4 min=32502 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=74 max_load_reduction_pct=0.002231 moved_to_replicas=74 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=9 tokens_per_ep_rank=[32457, 32519, 32502, 32908, 33170, 32812, 32797, 32979] avg=32768.0000 max=33170 max_rank=4 min=32457 min_rank=0 imbalance_ratio=1.012268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=9 tokens_per_ep_rank=[32457, 32519, 32502, 32908, 33170, 32812, 32797, 32979] avg=32768.0000 max=33170 max_rank=4 min=32457 min_rank=0 imbalance_ratio=1.012268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=10 tokens_per_ep_rank=[32615, 32608, 32627, 32757, 32817, 32842, 32885, 32993] avg=32768.0000 max=32993 max_rank=7 min=32608 min_rank=1 imbalance_ratio=1.006866 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=10 tokens_per_ep_rank=[32615, 32608, 32627, 32757, 32817, 32842, 32885, 32993] avg=32768.0000 max=32993 max_rank=7 min=32608 min_rank=1 imbalance_ratio=1.006866 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=9 tokens_per_ep_rank=[32457, 32519, 32502, 32908, 33170, 32812, 32797, 32979] avg=32768.0000 max=33170 max_rank=4 min=32457 min_rank=0 imbalance_ratio=1.012268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=10 tokens_per_ep_rank=[32615, 32608, 32627, 32757, 32817, 32842, 32885, 32993] avg=32768.0000 max=32993 max_rank=7 min=32608 min_rank=1 imbalance_ratio=1.006866 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=9 tokens_per_ep_rank=[32531, 32519, 32502, 32908, 33096, 32812, 32797, 32979] avg=32768.0000 max=33096 max_rank=4 min=32502 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=74 max_load_reduction_pct=0.002231 moved_to_replicas=74 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=9 tokens_per_ep_rank=[32531, 32519, 32502, 32908, 33096, 32812, 32797, 32979] avg=32768.0000 max=33096 max_rank=4 min=32502 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=74 max_load_reduction_pct=0.002231 moved_to_replicas=74 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=9 tokens_per_ep_rank=[32531, 32519, 32502, 32908, 33096, 32812, 32797, 32979] avg=32768.0000 max=33096 max_rank=4 min=32502 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=74 max_load_reduction_pct=0.002231 moved_to_replicas=74 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=10 tokens_per_ep_rank=[32615, 32608, 32627, 32757, 32817, 32842, 32885, 32993] avg=32768.0000 max=32993 max_rank=7 min=32608 min_rank=1 imbalance_ratio=1.006866 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=10 tokens_per_ep_rank=[32615, 32608, 32627, 32757, 32817, 32842, 32885, 32993] avg=32768.0000 max=32993 max_rank=7 min=32608 min_rank=1 imbalance_ratio=1.006866 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=10 tokens_per_ep_rank=[32615, 32608, 32627, 32757, 32817, 32842, 32885, 32993] avg=32768.0000 max=32993 max_rank=7 min=32608 min_rank=1 imbalance_ratio=1.006866 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=9 tokens_per_ep_rank=[32531, 32519, 32502, 32908, 33096, 32812, 32797, 32979] avg=32768.0000 max=33096 max_rank=4 min=32502 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=74 max_load_reduction_pct=0.002231 moved_to_replicas=74 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=10 tokens_per_ep_rank=[32615, 32608, 32627, 32757, 32817, 32842, 32885, 32993] avg=32768.0000 max=32993 max_rank=7 min=32608 min_rank=1 imbalance_ratio=1.006866 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=10 tokens_per_ep_rank=[32615, 32608, 32627, 32757, 32817, 32842, 32885, 32993] avg=32768.0000 max=32993 max_rank=7 min=32608 min_rank=1 imbalance_ratio=1.006866 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=10 tokens_per_ep_rank=[32615, 32608, 32627, 32757, 32817, 32842, 32885, 32993] avg=32768.0000 max=32993 max_rank=7 min=32608 min_rank=1 imbalance_ratio=1.006866 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=10 tokens_per_ep_rank=[32615, 32608, 32627, 32757, 32817, 32842, 32885, 32993] avg=32768.0000 max=32993 max_rank=7 min=32608 min_rank=1 imbalance_ratio=1.006866 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=10 tokens_per_ep_rank=[32615, 32608, 32627, 32757, 32817, 32842, 32885, 32993] avg=32768.0000 max=32993 max_rank=7 min=32608 min_rank=1 imbalance_ratio=1.006866 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=10 tokens_per_ep_rank=[32615, 32608, 32627, 32757, 32817, 32842, 32885, 32993] avg=32768.0000 max=32993 max_rank=7 min=32608 min_rank=1 imbalance_ratio=1.006866 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=10 tokens_per_ep_rank=[32615, 32608, 32627, 32757, 32817, 32842, 32885, 32993] avg=32768.0000 max=32993 max_rank=7 min=32608 min_rank=1 imbalance_ratio=1.006866 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=10 tokens_per_ep_rank=[32615, 32608, 32627, 32757, 32817, 32842, 32885, 32993] avg=32768.0000 max=32993 max_rank=7 min=32608 min_rank=1 imbalance_ratio=1.006866 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=10 tokens_per_ep_rank=[32615, 32608, 32627, 32757, 32817, 32842, 32885, 32993] avg=32768.0000 max=32993 max_rank=7 min=32608 min_rank=1 imbalance_ratio=1.006866 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=10 tokens_per_ep_rank=[32615, 32608, 32627, 32757, 32817, 32842, 32885, 32993] avg=32768.0000 max=32993 max_rank=7 min=32608 min_rank=1 imbalance_ratio=1.006866 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=2 records=10 total_expected_recv=328589 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76032 peak_demand=33036 required_capacity=1.008179 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=2 records=10 total_expected_recv=328409 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76032 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=2 records=10 total_expected_recv=327420 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76672 peak_demand=32924 required_capacity=1.004761 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=2 records=10 total_expected_recv=328020 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76032 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=2 records=10 total_expected_recv=326625 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=78208 peak_demand=32934 required_capacity=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=2 records=10 total_expected_recv=326893 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=77696 peak_demand=33031 required_capacity=1.008026 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=2 records=10 total_expected_recv=328114 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=75904 peak_demand=32979 required_capacity=1.006439 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=2 records=10 total_expected_recv=327370 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=77312 peak_demand=32923 required_capacity=1.004730 + [2026-08-26 15:53:12] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 215.7 | throughput per GPU (TFLOP/s/GPU): 572.5 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 9.516286E+00 | loss scale: 1.0 | grad norm: 30.353 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 33664 | min safe capacity: 1.0273 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=0 calls=1 budget=40960 expected_recv=32676 expected_overflow=0 exact_recv_per_rank=[32538, 32690, 32643, 33096, 32966, 32980, 32676, 32555] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=0 calls=1 budget=40960 expected_recv=32643 expected_overflow=0 exact_recv_per_rank=[32538, 32690, 32643, 33096, 32966, 32980, 32676, 32555] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=0 calls=1 budget=40960 expected_recv=32966 expected_overflow=0 exact_recv_per_rank=[32538, 32690, 32643, 33096, 32966, 32980, 32676, 32555] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=0 calls=1 budget=40960 expected_recv=32555 expected_overflow=0 exact_recv_per_rank=[32538, 32690, 32643, 33096, 32966, 32980, 32676, 32555] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=0 calls=1 budget=40960 expected_recv=32980 expected_overflow=0 exact_recv_per_rank=[32538, 32690, 32643, 33096, 32966, 32980, 32676, 32555] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32538, 32690, 32643, 33096, 32966, 32980, 32676, 32555] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=0 calls=1 budget=40960 expected_recv=32690 expected_overflow=0 exact_recv_per_rank=[32538, 32690, 32643, 33096, 32966, 32980, 32676, 32555] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=0 calls=1 budget=40960 expected_recv=32676 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=0 calls=1 budget=40960 expected_recv=32643 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=0 calls=1 budget=40960 expected_recv=32538 expected_overflow=0 exact_recv_per_rank=[32538, 32690, 32643, 33096, 32966, 32980, 32676, 32555] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=0 calls=1 budget=40960 expected_recv=32966 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=0 calls=1 budget=40960 expected_recv=32555 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=0 calls=1 budget=40960 expected_recv=32980 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=0 calls=1 budget=40960 expected_recv=32690 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=0 calls=1 budget=40960 expected_recv=32538 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=1 calls=1 budget=40960 expected_recv=32965 expected_overflow=0 exact_recv_per_rank=[32895, 32566, 32965, 32685, 32684, 32824, 32848, 32677] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=1 calls=1 budget=40960 expected_recv=32848 expected_overflow=0 exact_recv_per_rank=[32895, 32566, 32965, 32685, 32684, 32824, 32848, 32677] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=1 calls=1 budget=40960 expected_recv=32677 expected_overflow=0 exact_recv_per_rank=[32895, 32566, 32965, 32685, 32684, 32824, 32848, 32677] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=1 calls=1 budget=40960 expected_recv=32684 expected_overflow=0 exact_recv_per_rank=[32895, 32566, 32965, 32685, 32684, 32824, 32848, 32677] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=1 calls=1 budget=40960 expected_recv=32824 expected_overflow=0 exact_recv_per_rank=[32895, 32566, 32965, 32685, 32684, 32824, 32848, 32677] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=1 calls=1 budget=40960 expected_recv=32685 expected_overflow=0 exact_recv_per_rank=[32895, 32566, 32965, 32685, 32684, 32824, 32848, 32677] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=1 calls=1 budget=40960 expected_recv=32965 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=1 calls=1 budget=40960 expected_recv=32848 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=1 calls=1 budget=40960 expected_recv=32677 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=1 calls=1 budget=40960 expected_recv=32684 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=1 calls=1 budget=40960 expected_recv=32824 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=1 calls=1 budget=40960 expected_recv=32566 expected_overflow=0 exact_recv_per_rank=[32895, 32566, 32965, 32685, 32684, 32824, 32848, 32677] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=1 calls=1 budget=40960 expected_recv=32685 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=1 calls=1 budget=40960 expected_recv=32895 expected_overflow=0 exact_recv_per_rank=[32895, 32566, 32965, 32685, 32684, 32824, 32848, 32677] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=1 calls=1 budget=40960 expected_recv=32566 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=1 calls=1 budget=40960 expected_recv=32895 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=2 calls=1 budget=40960 expected_recv=32869 expected_overflow=0 exact_recv_per_rank=[32873, 32823, 32869, 32742, 32690, 32694, 32675, 32778] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=2 calls=1 budget=40960 expected_recv=32778 expected_overflow=0 exact_recv_per_rank=[32873, 32823, 32869, 32742, 32690, 32694, 32675, 32778] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=2 calls=1 budget=40960 expected_recv=32675 expected_overflow=0 exact_recv_per_rank=[32873, 32823, 32869, 32742, 32690, 32694, 32675, 32778] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=2 calls=1 budget=40960 expected_recv=32690 expected_overflow=0 exact_recv_per_rank=[32873, 32823, 32869, 32742, 32690, 32694, 32675, 32778] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=2 calls=1 budget=40960 expected_recv=32694 expected_overflow=0 exact_recv_per_rank=[32873, 32823, 32869, 32742, 32690, 32694, 32675, 32778] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=2 calls=1 budget=40960 expected_recv=32869 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=2 calls=1 budget=40960 expected_recv=32778 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=2 calls=1 budget=40960 expected_recv=32675 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=2 calls=1 budget=40960 expected_recv=32823 expected_overflow=0 exact_recv_per_rank=[32873, 32823, 32869, 32742, 32690, 32694, 32675, 32778] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=2 calls=1 budget=40960 expected_recv=32690 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=2 calls=1 budget=40960 expected_recv=32873 expected_overflow=0 exact_recv_per_rank=[32873, 32823, 32869, 32742, 32690, 32694, 32675, 32778] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=2 calls=1 budget=40960 expected_recv=32742 expected_overflow=0 exact_recv_per_rank=[32873, 32823, 32869, 32742, 32690, 32694, 32675, 32778] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=2 calls=1 budget=40960 expected_recv=32694 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=2 calls=1 budget=40960 expected_recv=32823 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=2 calls=1 budget=40960 expected_recv=32873 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=2 calls=1 budget=40960 expected_recv=32742 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=3 calls=1 budget=40960 expected_recv=32501 expected_overflow=0 exact_recv_per_rank=[32900, 32797, 32501, 32476, 32982, 32838, 32778, 32872] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=3 calls=1 budget=40960 expected_recv=32872 expected_overflow=0 exact_recv_per_rank=[32900, 32797, 32501, 32476, 32982, 32838, 32778, 32872] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=3 calls=1 budget=40960 expected_recv=32778 expected_overflow=0 exact_recv_per_rank=[32900, 32797, 32501, 32476, 32982, 32838, 32778, 32872] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=3 calls=1 budget=40960 expected_recv=32982 expected_overflow=0 exact_recv_per_rank=[32900, 32797, 32501, 32476, 32982, 32838, 32778, 32872] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=3 calls=1 budget=40960 expected_recv=32501 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=3 calls=1 budget=40960 expected_recv=32872 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=3 calls=1 budget=40960 expected_recv=32797 expected_overflow=0 exact_recv_per_rank=[32900, 32797, 32501, 32476, 32982, 32838, 32778, 32872] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=3 calls=1 budget=40960 expected_recv=32778 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=3 calls=1 budget=40960 expected_recv=32982 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=3 calls=1 budget=40960 expected_recv=32838 expected_overflow=0 exact_recv_per_rank=[32900, 32797, 32501, 32476, 32982, 32838, 32778, 32872] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=3 calls=1 budget=40960 expected_recv=32900 expected_overflow=0 exact_recv_per_rank=[32900, 32797, 32501, 32476, 32982, 32838, 32778, 32872] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=3 calls=1 budget=40960 expected_recv=32797 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=3 calls=1 budget=40960 expected_recv=32476 expected_overflow=0 exact_recv_per_rank=[32900, 32797, 32501, 32476, 32982, 32838, 32778, 32872] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=3 calls=1 budget=40960 expected_recv=32838 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=3 calls=1 budget=40960 expected_recv=32900 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=3 calls=1 budget=40960 expected_recv=32476 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=4 calls=1 budget=40960 expected_recv=32499 expected_overflow=0 exact_recv_per_rank=[32930, 32891, 32499, 32734, 33006, 32761, 32546, 32777] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=4 calls=1 budget=40960 expected_recv=32777 expected_overflow=0 exact_recv_per_rank=[32930, 32891, 32499, 32734, 33006, 32761, 32546, 32777] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=4 calls=1 budget=40960 expected_recv=32546 expected_overflow=0 exact_recv_per_rank=[32930, 32891, 32499, 32734, 33006, 32761, 32546, 32777] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=4 calls=1 budget=40960 expected_recv=33006 expected_overflow=0 exact_recv_per_rank=[32930, 32891, 32499, 32734, 33006, 32761, 32546, 32777] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=4 calls=1 budget=40960 expected_recv=32499 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=4 calls=1 budget=40960 expected_recv=32777 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=4 calls=1 budget=40960 expected_recv=33006 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=4 calls=1 budget=40960 expected_recv=32546 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=4 calls=1 budget=40960 expected_recv=32891 expected_overflow=0 exact_recv_per_rank=[32930, 32891, 32499, 32734, 33006, 32761, 32546, 32777] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=4 calls=1 budget=40960 expected_recv=32761 expected_overflow=0 exact_recv_per_rank=[32930, 32891, 32499, 32734, 33006, 32761, 32546, 32777] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=4 calls=1 budget=40960 expected_recv=32930 expected_overflow=0 exact_recv_per_rank=[32930, 32891, 32499, 32734, 33006, 32761, 32546, 32777] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=4 calls=1 budget=40960 expected_recv=32891 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=4 calls=1 budget=40960 expected_recv=32734 expected_overflow=0 exact_recv_per_rank=[32930, 32891, 32499, 32734, 33006, 32761, 32546, 32777] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=4 calls=1 budget=40960 expected_recv=32761 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=4 calls=1 budget=40960 expected_recv=32930 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=4 calls=1 budget=40960 expected_recv=32734 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=5 calls=1 budget=40960 expected_recv=32628 expected_overflow=0 exact_recv_per_rank=[32825, 32715, 32628, 32826, 32753, 32780, 32920, 32697] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=5 calls=1 budget=40960 expected_recv=32697 expected_overflow=0 exact_recv_per_rank=[32825, 32715, 32628, 32826, 32753, 32780, 32920, 32697] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=5 calls=1 budget=40960 expected_recv=32628 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=5 calls=1 budget=40960 expected_recv=32753 expected_overflow=0 exact_recv_per_rank=[32825, 32715, 32628, 32826, 32753, 32780, 32920, 32697] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=5 calls=1 budget=40960 expected_recv=32920 expected_overflow=0 exact_recv_per_rank=[32825, 32715, 32628, 32826, 32753, 32780, 32920, 32697] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=5 calls=1 budget=40960 expected_recv=32697 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=5 calls=1 budget=40960 expected_recv=32753 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=5 calls=1 budget=40960 expected_recv=32920 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=5 calls=1 budget=40960 expected_recv=32715 expected_overflow=0 exact_recv_per_rank=[32825, 32715, 32628, 32826, 32753, 32780, 32920, 32697] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=5 calls=1 budget=40960 expected_recv=32825 expected_overflow=0 exact_recv_per_rank=[32825, 32715, 32628, 32826, 32753, 32780, 32920, 32697] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=5 calls=1 budget=40960 expected_recv=32826 expected_overflow=0 exact_recv_per_rank=[32825, 32715, 32628, 32826, 32753, 32780, 32920, 32697] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=5 calls=1 budget=40960 expected_recv=32780 expected_overflow=0 exact_recv_per_rank=[32825, 32715, 32628, 32826, 32753, 32780, 32920, 32697] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=5 calls=1 budget=40960 expected_recv=32715 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=5 calls=1 budget=40960 expected_recv=32825 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=5 calls=1 budget=40960 expected_recv=32826 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=5 calls=1 budget=40960 expected_recv=32780 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=6 calls=1 budget=40960 expected_recv=32796 expected_overflow=0 exact_recv_per_rank=[32794, 32681, 32796, 32652, 32623, 32925, 32840, 32833] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=6 calls=1 budget=40960 expected_recv=32796 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=6 calls=1 budget=40960 expected_recv=32833 expected_overflow=0 exact_recv_per_rank=[32794, 32681, 32796, 32652, 32623, 32925, 32840, 32833] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=6 calls=1 budget=40960 expected_recv=32840 expected_overflow=0 exact_recv_per_rank=[32794, 32681, 32796, 32652, 32623, 32925, 32840, 32833] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=6 calls=1 budget=40960 expected_recv=32623 expected_overflow=0 exact_recv_per_rank=[32794, 32681, 32796, 32652, 32623, 32925, 32840, 32833] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=6 calls=1 budget=40960 expected_recv=32833 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=6 calls=1 budget=40960 expected_recv=32681 expected_overflow=0 exact_recv_per_rank=[32794, 32681, 32796, 32652, 32623, 32925, 32840, 32833] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=6 calls=1 budget=40960 expected_recv=32840 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=6 calls=1 budget=40960 expected_recv=32623 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=6 calls=1 budget=40960 expected_recv=32794 expected_overflow=0 exact_recv_per_rank=[32794, 32681, 32796, 32652, 32623, 32925, 32840, 32833] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=6 calls=1 budget=40960 expected_recv=32925 expected_overflow=0 exact_recv_per_rank=[32794, 32681, 32796, 32652, 32623, 32925, 32840, 32833] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=6 calls=1 budget=40960 expected_recv=32652 expected_overflow=0 exact_recv_per_rank=[32794, 32681, 32796, 32652, 32623, 32925, 32840, 32833] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=6 calls=1 budget=40960 expected_recv=32681 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=6 calls=1 budget=40960 expected_recv=32794 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=6 calls=1 budget=40960 expected_recv=32925 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=6 calls=1 budget=40960 expected_recv=32652 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=7 calls=1 budget=40960 expected_recv=32606 expected_overflow=0 exact_recv_per_rank=[32528, 32904, 32606, 32933, 32817, 32640, 32769, 32947] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=7 calls=1 budget=40960 expected_recv=32947 expected_overflow=0 exact_recv_per_rank=[32528, 32904, 32606, 32933, 32817, 32640, 32769, 32947] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=7 calls=1 budget=40960 expected_recv=32606 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=7 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[32528, 32904, 32606, 32933, 32817, 32640, 32769, 32947] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=7 calls=1 budget=40960 expected_recv=32947 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=7 calls=1 budget=40960 expected_recv=32817 expected_overflow=0 exact_recv_per_rank=[32528, 32904, 32606, 32933, 32817, 32640, 32769, 32947] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=7 calls=1 budget=40960 expected_recv=32904 expected_overflow=0 exact_recv_per_rank=[32528, 32904, 32606, 32933, 32817, 32640, 32769, 32947] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=7 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=7 calls=1 budget=40960 expected_recv=32817 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=7 calls=1 budget=40960 expected_recv=32528 expected_overflow=0 exact_recv_per_rank=[32528, 32904, 32606, 32933, 32817, 32640, 32769, 32947] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=7 calls=1 budget=40960 expected_recv=32640 expected_overflow=0 exact_recv_per_rank=[32528, 32904, 32606, 32933, 32817, 32640, 32769, 32947] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=7 calls=1 budget=40960 expected_recv=32904 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=7 calls=1 budget=40960 expected_recv=32528 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=7 calls=1 budget=40960 expected_recv=32933 expected_overflow=0 exact_recv_per_rank=[32528, 32904, 32606, 32933, 32817, 32640, 32769, 32947] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=7 calls=1 budget=40960 expected_recv=32640 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=8 calls=1 budget=40960 expected_recv=33072 expected_overflow=0 exact_recv_per_rank=[32558, 32800, 33072, 32851, 32497, 32592, 33058, 32716] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=7 calls=1 budget=40960 expected_recv=32933 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=8 calls=1 budget=40960 expected_recv=32716 expected_overflow=0 exact_recv_per_rank=[32558, 32800, 33072, 32851, 32497, 32592, 33058, 32716] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=8 calls=1 budget=40960 expected_recv=33072 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=8 calls=1 budget=40960 expected_recv=32716 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=8 calls=1 budget=40960 expected_recv=33058 expected_overflow=0 exact_recv_per_rank=[32558, 32800, 33072, 32851, 32497, 32592, 33058, 32716] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=8 calls=1 budget=40960 expected_recv=32497 expected_overflow=0 exact_recv_per_rank=[32558, 32800, 33072, 32851, 32497, 32592, 33058, 32716] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=8 calls=1 budget=40960 expected_recv=32497 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=8 calls=1 budget=40960 expected_recv=33058 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=8 calls=1 budget=40960 expected_recv=32800 expected_overflow=0 exact_recv_per_rank=[32558, 32800, 33072, 32851, 32497, 32592, 33058, 32716] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=8 calls=1 budget=40960 expected_recv=32558 expected_overflow=0 exact_recv_per_rank=[32558, 32800, 33072, 32851, 32497, 32592, 33058, 32716] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=8 calls=1 budget=40960 expected_recv=32800 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=8 calls=1 budget=40960 expected_recv=32592 expected_overflow=0 exact_recv_per_rank=[32558, 32800, 33072, 32851, 32497, 32592, 33058, 32716] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=8 calls=1 budget=40960 expected_recv=32558 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=8 calls=1 budget=40960 expected_recv=32851 expected_overflow=0 exact_recv_per_rank=[32558, 32800, 33072, 32851, 32497, 32592, 33058, 32716] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=9 calls=1 budget=40960 expected_recv=32532 expected_overflow=0 exact_recv_per_rank=[32870, 32768, 32532, 32966, 32686, 32843, 32650, 32829] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=8 calls=1 budget=40960 expected_recv=32592 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=8 calls=1 budget=40960 expected_recv=32851 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=9 calls=1 budget=40960 expected_recv=32829 expected_overflow=0 exact_recv_per_rank=[32870, 32768, 32532, 32966, 32686, 32843, 32650, 32829] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=9 calls=1 budget=40960 expected_recv=32532 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=9 calls=1 budget=40960 expected_recv=32829 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=9 calls=1 budget=40960 expected_recv=32686 expected_overflow=0 exact_recv_per_rank=[32870, 32768, 32532, 32966, 32686, 32843, 32650, 32829] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=9 calls=1 budget=40960 expected_recv=32650 expected_overflow=0 exact_recv_per_rank=[32870, 32768, 32532, 32966, 32686, 32843, 32650, 32829] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=9 calls=1 budget=40960 expected_recv=32686 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=9 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32870, 32768, 32532, 32966, 32686, 32843, 32650, 32829] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=9 calls=1 budget=40960 expected_recv=32870 expected_overflow=0 exact_recv_per_rank=[32870, 32768, 32532, 32966, 32686, 32843, 32650, 32829] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=9 calls=1 budget=40960 expected_recv=32650 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=9 calls=1 budget=40960 expected_recv=32843 expected_overflow=0 exact_recv_per_rank=[32870, 32768, 32532, 32966, 32686, 32843, 32650, 32829] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=9 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=9 calls=1 budget=40960 expected_recv=32966 expected_overflow=0 exact_recv_per_rank=[32870, 32768, 32532, 32966, 32686, 32843, 32650, 32829] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=9 calls=1 budget=40960 expected_recv=32870 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=9 calls=1 budget=40960 expected_recv=32843 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=9 calls=1 budget=40960 expected_recv=32966 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=1 tokens_per_ep_rank=[32538, 32690, 32643, 33124, 32966, 32980, 32676, 32527] avg=32768.0000 max=33124 max_rank=3 min=32527 min_rank=7 imbalance_ratio=1.010864 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=1 tokens_per_ep_rank=[32538, 32690, 32643, 33124, 32966, 32980, 32676, 32527] avg=32768.0000 max=33124 max_rank=3 min=32527 min_rank=7 imbalance_ratio=1.010864 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=1 tokens_per_ep_rank=[32538, 32690, 32643, 33124, 32966, 32980, 32676, 32527] avg=32768.0000 max=33124 max_rank=3 min=32527 min_rank=7 imbalance_ratio=1.010864 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=1 tokens_per_ep_rank=[32538, 32690, 32643, 33096, 32966, 32980, 32676, 32555] avg=32768.0000 max=33096 max_rank=3 min=32538 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=28 max_load_reduction_pct=0.000845 moved_to_replicas=28 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=1 tokens_per_ep_rank=[32538, 32690, 32643, 33124, 32966, 32980, 32676, 32527] avg=32768.0000 max=33124 max_rank=3 min=32527 min_rank=7 imbalance_ratio=1.010864 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=1 tokens_per_ep_rank=[32538, 32690, 32643, 33124, 32966, 32980, 32676, 32527] avg=32768.0000 max=33124 max_rank=3 min=32527 min_rank=7 imbalance_ratio=1.010864 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=1 tokens_per_ep_rank=[32538, 32690, 32643, 33096, 32966, 32980, 32676, 32555] avg=32768.0000 max=33096 max_rank=3 min=32538 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=28 max_load_reduction_pct=0.000845 moved_to_replicas=28 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=1 tokens_per_ep_rank=[32538, 32690, 32643, 33124, 32966, 32980, 32676, 32527] avg=32768.0000 max=33124 max_rank=3 min=32527 min_rank=7 imbalance_ratio=1.010864 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=1 tokens_per_ep_rank=[32538, 32690, 32643, 33124, 32966, 32980, 32676, 32527] avg=32768.0000 max=33124 max_rank=3 min=32527 min_rank=7 imbalance_ratio=1.010864 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=1 tokens_per_ep_rank=[32538, 32690, 32643, 33124, 32966, 32980, 32676, 32527] avg=32768.0000 max=33124 max_rank=3 min=32527 min_rank=7 imbalance_ratio=1.010864 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=1 tokens_per_ep_rank=[32538, 32690, 32643, 33096, 32966, 32980, 32676, 32555] avg=32768.0000 max=33096 max_rank=3 min=32538 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=28 max_load_reduction_pct=0.000845 moved_to_replicas=28 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=1 tokens_per_ep_rank=[32538, 32690, 32643, 33096, 32966, 32980, 32676, 32555] avg=32768.0000 max=33096 max_rank=3 min=32538 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=28 max_load_reduction_pct=0.000845 moved_to_replicas=28 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=1 tokens_per_ep_rank=[32538, 32690, 32643, 33096, 32966, 32980, 32676, 32555] avg=32768.0000 max=33096 max_rank=3 min=32538 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=28 max_load_reduction_pct=0.000845 moved_to_replicas=28 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=1 tokens_per_ep_rank=[32538, 32690, 32643, 33096, 32966, 32980, 32676, 32555] avg=32768.0000 max=33096 max_rank=3 min=32538 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=28 max_load_reduction_pct=0.000845 moved_to_replicas=28 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=2 tokens_per_ep_rank=[32895, 32566, 32965, 32685, 32684, 32824, 32848, 32677] avg=32768.0000 max=32965 max_rank=2 min=32566 min_rank=1 imbalance_ratio=1.006012 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=1 tokens_per_ep_rank=[32538, 32690, 32643, 33096, 32966, 32980, 32676, 32555] avg=32768.0000 max=33096 max_rank=3 min=32538 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=28 max_load_reduction_pct=0.000845 moved_to_replicas=28 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=1 tokens_per_ep_rank=[32538, 32690, 32643, 33096, 32966, 32980, 32676, 32555] avg=32768.0000 max=33096 max_rank=3 min=32538 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=28 max_load_reduction_pct=0.000845 moved_to_replicas=28 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=2 tokens_per_ep_rank=[32895, 32566, 32965, 32685, 32684, 32824, 32848, 32677] avg=32768.0000 max=32965 max_rank=2 min=32566 min_rank=1 imbalance_ratio=1.006012 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=2 tokens_per_ep_rank=[32895, 32566, 32965, 32685, 32684, 32824, 32848, 32677] avg=32768.0000 max=32965 max_rank=2 min=32566 min_rank=1 imbalance_ratio=1.006012 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=2 tokens_per_ep_rank=[32895, 32566, 32965, 32685, 32684, 32824, 32848, 32677] avg=32768.0000 max=32965 max_rank=2 min=32566 min_rank=1 imbalance_ratio=1.006012 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=2 tokens_per_ep_rank=[32895, 32566, 32965, 32685, 32684, 32824, 32848, 32677] avg=32768.0000 max=32965 max_rank=2 min=32566 min_rank=1 imbalance_ratio=1.006012 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=2 tokens_per_ep_rank=[32895, 32566, 32965, 32685, 32684, 32824, 32848, 32677] avg=32768.0000 max=32965 max_rank=2 min=32566 min_rank=1 imbalance_ratio=1.006012 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=2 tokens_per_ep_rank=[32895, 32566, 32965, 32685, 32684, 32824, 32848, 32677] avg=32768.0000 max=32965 max_rank=2 min=32566 min_rank=1 imbalance_ratio=1.006012 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=2 tokens_per_ep_rank=[32895, 32566, 32965, 32685, 32684, 32824, 32848, 32677] avg=32768.0000 max=32965 max_rank=2 min=32566 min_rank=1 imbalance_ratio=1.006012 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=2 tokens_per_ep_rank=[32895, 32566, 32965, 32685, 32684, 32824, 32848, 32677] avg=32768.0000 max=32965 max_rank=2 min=32566 min_rank=1 imbalance_ratio=1.006012 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=2 tokens_per_ep_rank=[32895, 32566, 32965, 32685, 32684, 32824, 32848, 32677] avg=32768.0000 max=32965 max_rank=2 min=32566 min_rank=1 imbalance_ratio=1.006012 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=2 tokens_per_ep_rank=[32895, 32566, 32965, 32685, 32684, 32824, 32848, 32677] avg=32768.0000 max=32965 max_rank=2 min=32566 min_rank=1 imbalance_ratio=1.006012 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=2 tokens_per_ep_rank=[32895, 32566, 32965, 32685, 32684, 32824, 32848, 32677] avg=32768.0000 max=32965 max_rank=2 min=32566 min_rank=1 imbalance_ratio=1.006012 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=2 tokens_per_ep_rank=[32895, 32566, 32965, 32685, 32684, 32824, 32848, 32677] avg=32768.0000 max=32965 max_rank=2 min=32566 min_rank=1 imbalance_ratio=1.006012 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=3 tokens_per_ep_rank=[32873, 32823, 32869, 32742, 32690, 32694, 32675, 32778] avg=32768.0000 max=32873 max_rank=0 min=32675 min_rank=6 imbalance_ratio=1.003204 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=3 tokens_per_ep_rank=[32873, 32823, 32869, 32742, 32690, 32694, 32675, 32778] avg=32768.0000 max=32873 max_rank=0 min=32675 min_rank=6 imbalance_ratio=1.003204 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=2 tokens_per_ep_rank=[32895, 32566, 32965, 32685, 32684, 32824, 32848, 32677] avg=32768.0000 max=32965 max_rank=2 min=32566 min_rank=1 imbalance_ratio=1.006012 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=2 tokens_per_ep_rank=[32895, 32566, 32965, 32685, 32684, 32824, 32848, 32677] avg=32768.0000 max=32965 max_rank=2 min=32566 min_rank=1 imbalance_ratio=1.006012 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=2 tokens_per_ep_rank=[32895, 32566, 32965, 32685, 32684, 32824, 32848, 32677] avg=32768.0000 max=32965 max_rank=2 min=32566 min_rank=1 imbalance_ratio=1.006012 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=3 tokens_per_ep_rank=[32873, 32823, 32869, 32742, 32690, 32694, 32675, 32778] avg=32768.0000 max=32873 max_rank=0 min=32675 min_rank=6 imbalance_ratio=1.003204 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=3 tokens_per_ep_rank=[32873, 32823, 32869, 32742, 32690, 32694, 32675, 32778] avg=32768.0000 max=32873 max_rank=0 min=32675 min_rank=6 imbalance_ratio=1.003204 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=3 tokens_per_ep_rank=[32873, 32823, 32869, 32742, 32690, 32694, 32675, 32778] avg=32768.0000 max=32873 max_rank=0 min=32675 min_rank=6 imbalance_ratio=1.003204 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=3 tokens_per_ep_rank=[32873, 32823, 32869, 32742, 32690, 32694, 32675, 32778] avg=32768.0000 max=32873 max_rank=0 min=32675 min_rank=6 imbalance_ratio=1.003204 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=3 tokens_per_ep_rank=[32873, 32823, 32869, 32742, 32690, 32694, 32675, 32778] avg=32768.0000 max=32873 max_rank=0 min=32675 min_rank=6 imbalance_ratio=1.003204 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=3 tokens_per_ep_rank=[32873, 32823, 32869, 32742, 32690, 32694, 32675, 32778] avg=32768.0000 max=32873 max_rank=0 min=32675 min_rank=6 imbalance_ratio=1.003204 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=3 tokens_per_ep_rank=[32873, 32823, 32869, 32742, 32690, 32694, 32675, 32778] avg=32768.0000 max=32873 max_rank=0 min=32675 min_rank=6 imbalance_ratio=1.003204 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=3 tokens_per_ep_rank=[32873, 32823, 32869, 32742, 32690, 32694, 32675, 32778] avg=32768.0000 max=32873 max_rank=0 min=32675 min_rank=6 imbalance_ratio=1.003204 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=3 tokens_per_ep_rank=[32873, 32823, 32869, 32742, 32690, 32694, 32675, 32778] avg=32768.0000 max=32873 max_rank=0 min=32675 min_rank=6 imbalance_ratio=1.003204 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=3 tokens_per_ep_rank=[32873, 32823, 32869, 32742, 32690, 32694, 32675, 32778] avg=32768.0000 max=32873 max_rank=0 min=32675 min_rank=6 imbalance_ratio=1.003204 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=3 tokens_per_ep_rank=[32873, 32823, 32869, 32742, 32690, 32694, 32675, 32778] avg=32768.0000 max=32873 max_rank=0 min=32675 min_rank=6 imbalance_ratio=1.003204 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=4 tokens_per_ep_rank=[32900, 32797, 32501, 32476, 32982, 32838, 32778, 32872] avg=32768.0000 max=32982 max_rank=4 min=32476 min_rank=3 imbalance_ratio=1.006531 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=4 tokens_per_ep_rank=[32900, 32797, 32501, 32476, 32982, 32838, 32778, 32872] avg=32768.0000 max=32982 max_rank=4 min=32476 min_rank=3 imbalance_ratio=1.006531 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=3 tokens_per_ep_rank=[32873, 32823, 32869, 32742, 32690, 32694, 32675, 32778] avg=32768.0000 max=32873 max_rank=0 min=32675 min_rank=6 imbalance_ratio=1.003204 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=3 tokens_per_ep_rank=[32873, 32823, 32869, 32742, 32690, 32694, 32675, 32778] avg=32768.0000 max=32873 max_rank=0 min=32675 min_rank=6 imbalance_ratio=1.003204 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=3 tokens_per_ep_rank=[32873, 32823, 32869, 32742, 32690, 32694, 32675, 32778] avg=32768.0000 max=32873 max_rank=0 min=32675 min_rank=6 imbalance_ratio=1.003204 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=4 tokens_per_ep_rank=[32900, 32797, 32501, 32476, 32982, 32838, 32778, 32872] avg=32768.0000 max=32982 max_rank=4 min=32476 min_rank=3 imbalance_ratio=1.006531 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=4 tokens_per_ep_rank=[32900, 32797, 32501, 32476, 32982, 32838, 32778, 32872] avg=32768.0000 max=32982 max_rank=4 min=32476 min_rank=3 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=4 tokens_per_ep_rank=[32900, 32797, 32501, 32476, 32982, 32838, 32778, 32872] avg=32768.0000 max=32982 max_rank=4 min=32476 min_rank=3 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=4 tokens_per_ep_rank=[32900, 32797, 32501, 32476, 32982, 32838, 32778, 32872] avg=32768.0000 max=32982 max_rank=4 min=32476 min_rank=3 imbalance_ratio=1.006531 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=4 tokens_per_ep_rank=[32900, 32797, 32501, 32476, 32982, 32838, 32778, 32872] avg=32768.0000 max=32982 max_rank=4 min=32476 min_rank=3 imbalance_ratio=1.006531 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=4 tokens_per_ep_rank=[32900, 32797, 32501, 32476, 32982, 32838, 32778, 32872] avg=32768.0000 max=32982 max_rank=4 min=32476 min_rank=3 imbalance_ratio=1.006531 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=4 tokens_per_ep_rank=[32900, 32797, 32501, 32476, 32982, 32838, 32778, 32872] avg=32768.0000 max=32982 max_rank=4 min=32476 min_rank=3 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=4 tokens_per_ep_rank=[32900, 32797, 32501, 32476, 32982, 32838, 32778, 32872] avg=32768.0000 max=32982 max_rank=4 min=32476 min_rank=3 imbalance_ratio=1.006531 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=4 tokens_per_ep_rank=[32900, 32797, 32501, 32476, 32982, 32838, 32778, 32872] avg=32768.0000 max=32982 max_rank=4 min=32476 min_rank=3 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=4 tokens_per_ep_rank=[32900, 32797, 32501, 32476, 32982, 32838, 32778, 32872] avg=32768.0000 max=32982 max_rank=4 min=32476 min_rank=3 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=5 tokens_per_ep_rank=[32930, 32891, 32499, 32734, 33006, 32761, 32546, 32777] avg=32768.0000 max=33006 max_rank=4 min=32499 min_rank=2 imbalance_ratio=1.007263 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=4 tokens_per_ep_rank=[32900, 32797, 32501, 32476, 32982, 32838, 32778, 32872] avg=32768.0000 max=32982 max_rank=4 min=32476 min_rank=3 imbalance_ratio=1.006531 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=5 tokens_per_ep_rank=[32930, 32891, 32499, 32734, 33006, 32761, 32546, 32777] avg=32768.0000 max=33006 max_rank=4 min=32499 min_rank=2 imbalance_ratio=1.007263 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=4 tokens_per_ep_rank=[32900, 32797, 32501, 32476, 32982, 32838, 32778, 32872] avg=32768.0000 max=32982 max_rank=4 min=32476 min_rank=3 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=4 tokens_per_ep_rank=[32900, 32797, 32501, 32476, 32982, 32838, 32778, 32872] avg=32768.0000 max=32982 max_rank=4 min=32476 min_rank=3 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=5 tokens_per_ep_rank=[32930, 32891, 32499, 32734, 33006, 32761, 32546, 32777] avg=32768.0000 max=33006 max_rank=4 min=32499 min_rank=2 imbalance_ratio=1.007263 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=5 tokens_per_ep_rank=[32930, 32891, 32499, 32734, 33006, 32761, 32546, 32777] avg=32768.0000 max=33006 max_rank=4 min=32499 min_rank=2 imbalance_ratio=1.007263 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=5 tokens_per_ep_rank=[32930, 32891, 32499, 32734, 33006, 32761, 32546, 32777] avg=32768.0000 max=33006 max_rank=4 min=32499 min_rank=2 imbalance_ratio=1.007263 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=4 tokens_per_ep_rank=[32900, 32797, 32501, 32476, 32982, 32838, 32778, 32872] avg=32768.0000 max=32982 max_rank=4 min=32476 min_rank=3 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=5 tokens_per_ep_rank=[32930, 32891, 32499, 32734, 33006, 32761, 32546, 32777] avg=32768.0000 max=33006 max_rank=4 min=32499 min_rank=2 imbalance_ratio=1.007263 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=5 tokens_per_ep_rank=[32930, 32891, 32499, 32734, 33006, 32761, 32546, 32777] avg=32768.0000 max=33006 max_rank=4 min=32499 min_rank=2 imbalance_ratio=1.007263 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=5 tokens_per_ep_rank=[32930, 32891, 32499, 32734, 33006, 32761, 32546, 32777] avg=32768.0000 max=33006 max_rank=4 min=32499 min_rank=2 imbalance_ratio=1.007263 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=5 tokens_per_ep_rank=[32930, 32891, 32499, 32734, 33006, 32761, 32546, 32777] avg=32768.0000 max=33006 max_rank=4 min=32499 min_rank=2 imbalance_ratio=1.007263 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=5 tokens_per_ep_rank=[32930, 32891, 32499, 32734, 33006, 32761, 32546, 32777] avg=32768.0000 max=33006 max_rank=4 min=32499 min_rank=2 imbalance_ratio=1.007263 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=5 tokens_per_ep_rank=[32930, 32891, 32499, 32734, 33006, 32761, 32546, 32777] avg=32768.0000 max=33006 max_rank=4 min=32499 min_rank=2 imbalance_ratio=1.007263 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=5 tokens_per_ep_rank=[32930, 32891, 32499, 32734, 33006, 32761, 32546, 32777] avg=32768.0000 max=33006 max_rank=4 min=32499 min_rank=2 imbalance_ratio=1.007263 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=6 tokens_per_ep_rank=[32825, 32715, 32628, 32826, 32753, 32780, 32920, 32697] avg=32768.0000 max=32920 max_rank=6 min=32628 min_rank=2 imbalance_ratio=1.004639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=6 tokens_per_ep_rank=[32825, 32715, 32628, 32826, 32753, 32780, 32920, 32697] avg=32768.0000 max=32920 max_rank=6 min=32628 min_rank=2 imbalance_ratio=1.004639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=5 tokens_per_ep_rank=[32930, 32891, 32499, 32734, 33006, 32761, 32546, 32777] avg=32768.0000 max=33006 max_rank=4 min=32499 min_rank=2 imbalance_ratio=1.007263 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=5 tokens_per_ep_rank=[32930, 32891, 32499, 32734, 33006, 32761, 32546, 32777] avg=32768.0000 max=33006 max_rank=4 min=32499 min_rank=2 imbalance_ratio=1.007263 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=5 tokens_per_ep_rank=[32930, 32891, 32499, 32734, 33006, 32761, 32546, 32777] avg=32768.0000 max=33006 max_rank=4 min=32499 min_rank=2 imbalance_ratio=1.007263 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=6 tokens_per_ep_rank=[32825, 32715, 32628, 32826, 32753, 32780, 32920, 32697] avg=32768.0000 max=32920 max_rank=6 min=32628 min_rank=2 imbalance_ratio=1.004639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=6 tokens_per_ep_rank=[32825, 32715, 32628, 32826, 32753, 32780, 32920, 32697] avg=32768.0000 max=32920 max_rank=6 min=32628 min_rank=2 imbalance_ratio=1.004639 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=6 tokens_per_ep_rank=[32825, 32715, 32628, 32826, 32753, 32780, 32920, 32697] avg=32768.0000 max=32920 max_rank=6 min=32628 min_rank=2 imbalance_ratio=1.004639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=6 tokens_per_ep_rank=[32825, 32715, 32628, 32826, 32753, 32780, 32920, 32697] avg=32768.0000 max=32920 max_rank=6 min=32628 min_rank=2 imbalance_ratio=1.004639 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=6 tokens_per_ep_rank=[32825, 32715, 32628, 32826, 32753, 32780, 32920, 32697] avg=32768.0000 max=32920 max_rank=6 min=32628 min_rank=2 imbalance_ratio=1.004639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=5 tokens_per_ep_rank=[32930, 32891, 32499, 32734, 33006, 32761, 32546, 32777] avg=32768.0000 max=33006 max_rank=4 min=32499 min_rank=2 imbalance_ratio=1.007263 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=6 tokens_per_ep_rank=[32825, 32715, 32628, 32826, 32753, 32780, 32920, 32697] avg=32768.0000 max=32920 max_rank=6 min=32628 min_rank=2 imbalance_ratio=1.004639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=6 tokens_per_ep_rank=[32825, 32715, 32628, 32826, 32753, 32780, 32920, 32697] avg=32768.0000 max=32920 max_rank=6 min=32628 min_rank=2 imbalance_ratio=1.004639 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=6 tokens_per_ep_rank=[32825, 32715, 32628, 32826, 32753, 32780, 32920, 32697] avg=32768.0000 max=32920 max_rank=6 min=32628 min_rank=2 imbalance_ratio=1.004639 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=6 tokens_per_ep_rank=[32825, 32715, 32628, 32826, 32753, 32780, 32920, 32697] avg=32768.0000 max=32920 max_rank=6 min=32628 min_rank=2 imbalance_ratio=1.004639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=6 tokens_per_ep_rank=[32825, 32715, 32628, 32826, 32753, 32780, 32920, 32697] avg=32768.0000 max=32920 max_rank=6 min=32628 min_rank=2 imbalance_ratio=1.004639 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=7 tokens_per_ep_rank=[32794, 32681, 32796, 32652, 32623, 32925, 32840, 32833] avg=32768.0000 max=32925 max_rank=5 min=32623 min_rank=4 imbalance_ratio=1.004791 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=7 tokens_per_ep_rank=[32794, 32681, 32796, 32652, 32623, 32925, 32840, 32833] avg=32768.0000 max=32925 max_rank=5 min=32623 min_rank=4 imbalance_ratio=1.004791 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=6 tokens_per_ep_rank=[32825, 32715, 32628, 32826, 32753, 32780, 32920, 32697] avg=32768.0000 max=32920 max_rank=6 min=32628 min_rank=2 imbalance_ratio=1.004639 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=6 tokens_per_ep_rank=[32825, 32715, 32628, 32826, 32753, 32780, 32920, 32697] avg=32768.0000 max=32920 max_rank=6 min=32628 min_rank=2 imbalance_ratio=1.004639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=7 tokens_per_ep_rank=[32794, 32681, 32796, 32652, 32623, 32925, 32840, 32833] avg=32768.0000 max=32925 max_rank=5 min=32623 min_rank=4 imbalance_ratio=1.004791 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=6 tokens_per_ep_rank=[32825, 32715, 32628, 32826, 32753, 32780, 32920, 32697] avg=32768.0000 max=32920 max_rank=6 min=32628 min_rank=2 imbalance_ratio=1.004639 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=7 tokens_per_ep_rank=[32794, 32681, 32796, 32652, 32623, 32925, 32840, 32833] avg=32768.0000 max=32925 max_rank=5 min=32623 min_rank=4 imbalance_ratio=1.004791 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=7 tokens_per_ep_rank=[32794, 32681, 32796, 32652, 32623, 32925, 32840, 32833] avg=32768.0000 max=32925 max_rank=5 min=32623 min_rank=4 imbalance_ratio=1.004791 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=7 tokens_per_ep_rank=[32794, 32681, 32796, 32652, 32623, 32925, 32840, 32833] avg=32768.0000 max=32925 max_rank=5 min=32623 min_rank=4 imbalance_ratio=1.004791 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=7 tokens_per_ep_rank=[32794, 32681, 32796, 32652, 32623, 32925, 32840, 32833] avg=32768.0000 max=32925 max_rank=5 min=32623 min_rank=4 imbalance_ratio=1.004791 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=6 tokens_per_ep_rank=[32825, 32715, 32628, 32826, 32753, 32780, 32920, 32697] avg=32768.0000 max=32920 max_rank=6 min=32628 min_rank=2 imbalance_ratio=1.004639 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=7 tokens_per_ep_rank=[32794, 32681, 32796, 32652, 32623, 32925, 32840, 32833] avg=32768.0000 max=32925 max_rank=5 min=32623 min_rank=4 imbalance_ratio=1.004791 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=7 tokens_per_ep_rank=[32794, 32681, 32796, 32652, 32623, 32925, 32840, 32833] avg=32768.0000 max=32925 max_rank=5 min=32623 min_rank=4 imbalance_ratio=1.004791 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=7 tokens_per_ep_rank=[32794, 32681, 32796, 32652, 32623, 32925, 32840, 32833] avg=32768.0000 max=32925 max_rank=5 min=32623 min_rank=4 imbalance_ratio=1.004791 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=8 tokens_per_ep_rank=[32528, 32904, 32606, 32933, 32817, 32640, 32769, 32947] avg=32768.0000 max=32947 max_rank=7 min=32528 min_rank=0 imbalance_ratio=1.005463 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=7 tokens_per_ep_rank=[32794, 32681, 32796, 32652, 32623, 32925, 32840, 32833] avg=32768.0000 max=32925 max_rank=5 min=32623 min_rank=4 imbalance_ratio=1.004791 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=7 tokens_per_ep_rank=[32794, 32681, 32796, 32652, 32623, 32925, 32840, 32833] avg=32768.0000 max=32925 max_rank=5 min=32623 min_rank=4 imbalance_ratio=1.004791 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=8 tokens_per_ep_rank=[32528, 32904, 32606, 32933, 32817, 32640, 32769, 32947] avg=32768.0000 max=32947 max_rank=7 min=32528 min_rank=0 imbalance_ratio=1.005463 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=7 tokens_per_ep_rank=[32794, 32681, 32796, 32652, 32623, 32925, 32840, 32833] avg=32768.0000 max=32925 max_rank=5 min=32623 min_rank=4 imbalance_ratio=1.004791 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=8 tokens_per_ep_rank=[32528, 32904, 32606, 32933, 32817, 32640, 32769, 32947] avg=32768.0000 max=32947 max_rank=7 min=32528 min_rank=0 imbalance_ratio=1.005463 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=7 tokens_per_ep_rank=[32794, 32681, 32796, 32652, 32623, 32925, 32840, 32833] avg=32768.0000 max=32925 max_rank=5 min=32623 min_rank=4 imbalance_ratio=1.004791 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=7 tokens_per_ep_rank=[32794, 32681, 32796, 32652, 32623, 32925, 32840, 32833] avg=32768.0000 max=32925 max_rank=5 min=32623 min_rank=4 imbalance_ratio=1.004791 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=8 tokens_per_ep_rank=[32528, 32904, 32606, 32933, 32817, 32640, 32769, 32947] avg=32768.0000 max=32947 max_rank=7 min=32528 min_rank=0 imbalance_ratio=1.005463 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=8 tokens_per_ep_rank=[32528, 32904, 32606, 32933, 32817, 32640, 32769, 32947] avg=32768.0000 max=32947 max_rank=7 min=32528 min_rank=0 imbalance_ratio=1.005463 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=8 tokens_per_ep_rank=[32528, 32904, 32606, 32933, 32817, 32640, 32769, 32947] avg=32768.0000 max=32947 max_rank=7 min=32528 min_rank=0 imbalance_ratio=1.005463 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=8 tokens_per_ep_rank=[32528, 32904, 32606, 32933, 32817, 32640, 32769, 32947] avg=32768.0000 max=32947 max_rank=7 min=32528 min_rank=0 imbalance_ratio=1.005463 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=7 tokens_per_ep_rank=[32794, 32681, 32796, 32652, 32623, 32925, 32840, 32833] avg=32768.0000 max=32925 max_rank=5 min=32623 min_rank=4 imbalance_ratio=1.004791 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=8 tokens_per_ep_rank=[32528, 32904, 32606, 32933, 32817, 32640, 32769, 32947] avg=32768.0000 max=32947 max_rank=7 min=32528 min_rank=0 imbalance_ratio=1.005463 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=8 tokens_per_ep_rank=[32528, 32904, 32606, 32933, 32817, 32640, 32769, 32947] avg=32768.0000 max=32947 max_rank=7 min=32528 min_rank=0 imbalance_ratio=1.005463 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=8 tokens_per_ep_rank=[32528, 32904, 32606, 32933, 32817, 32640, 32769, 32947] avg=32768.0000 max=32947 max_rank=7 min=32528 min_rank=0 imbalance_ratio=1.005463 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=9 tokens_per_ep_rank=[32558, 32800, 33072, 32851, 32497, 32592, 33058, 32716] avg=32768.0000 max=33072 max_rank=2 min=32497 min_rank=4 imbalance_ratio=1.009277 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=8 tokens_per_ep_rank=[32528, 32904, 32606, 32933, 32817, 32640, 32769, 32947] avg=32768.0000 max=32947 max_rank=7 min=32528 min_rank=0 imbalance_ratio=1.005463 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=8 tokens_per_ep_rank=[32528, 32904, 32606, 32933, 32817, 32640, 32769, 32947] avg=32768.0000 max=32947 max_rank=7 min=32528 min_rank=0 imbalance_ratio=1.005463 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=9 tokens_per_ep_rank=[32558, 32800, 33072, 32851, 32497, 32592, 33058, 32716] avg=32768.0000 max=33072 max_rank=2 min=32497 min_rank=4 imbalance_ratio=1.009277 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=8 tokens_per_ep_rank=[32528, 32904, 32606, 32933, 32817, 32640, 32769, 32947] avg=32768.0000 max=32947 max_rank=7 min=32528 min_rank=0 imbalance_ratio=1.005463 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=9 tokens_per_ep_rank=[32558, 32800, 33072, 32851, 32497, 32592, 33058, 32716] avg=32768.0000 max=33072 max_rank=2 min=32497 min_rank=4 imbalance_ratio=1.009277 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=8 tokens_per_ep_rank=[32528, 32904, 32606, 32933, 32817, 32640, 32769, 32947] avg=32768.0000 max=32947 max_rank=7 min=32528 min_rank=0 imbalance_ratio=1.005463 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=8 tokens_per_ep_rank=[32528, 32904, 32606, 32933, 32817, 32640, 32769, 32947] avg=32768.0000 max=32947 max_rank=7 min=32528 min_rank=0 imbalance_ratio=1.005463 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=9 tokens_per_ep_rank=[32558, 32800, 33072, 32851, 32497, 32592, 33058, 32716] avg=32768.0000 max=33072 max_rank=2 min=32497 min_rank=4 imbalance_ratio=1.009277 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=9 tokens_per_ep_rank=[32558, 32800, 33072, 32851, 32497, 32592, 33058, 32716] avg=32768.0000 max=33072 max_rank=2 min=32497 min_rank=4 imbalance_ratio=1.009277 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=9 tokens_per_ep_rank=[32558, 32800, 33072, 32851, 32497, 32592, 33058, 32716] avg=32768.0000 max=33072 max_rank=2 min=32497 min_rank=4 imbalance_ratio=1.009277 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=9 tokens_per_ep_rank=[32558, 32800, 33072, 32851, 32497, 32592, 33058, 32716] avg=32768.0000 max=33072 max_rank=2 min=32497 min_rank=4 imbalance_ratio=1.009277 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=9 tokens_per_ep_rank=[32558, 32800, 33072, 32851, 32497, 32592, 33058, 32716] avg=32768.0000 max=33072 max_rank=2 min=32497 min_rank=4 imbalance_ratio=1.009277 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=8 tokens_per_ep_rank=[32528, 32904, 32606, 32933, 32817, 32640, 32769, 32947] avg=32768.0000 max=32947 max_rank=7 min=32528 min_rank=0 imbalance_ratio=1.005463 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=9 tokens_per_ep_rank=[32558, 32800, 33072, 32851, 32497, 32592, 33058, 32716] avg=32768.0000 max=33072 max_rank=2 min=32497 min_rank=4 imbalance_ratio=1.009277 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=9 tokens_per_ep_rank=[32558, 32800, 33072, 32851, 32497, 32592, 33058, 32716] avg=32768.0000 max=33072 max_rank=2 min=32497 min_rank=4 imbalance_ratio=1.009277 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=10 tokens_per_ep_rank=[32870, 32768, 32532, 32966, 32686, 32843, 32650, 32829] avg=32768.0000 max=32966 max_rank=3 min=32532 min_rank=2 imbalance_ratio=1.006042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=9 tokens_per_ep_rank=[32558, 32800, 33072, 32851, 32497, 32592, 33058, 32716] avg=32768.0000 max=33072 max_rank=2 min=32497 min_rank=4 imbalance_ratio=1.009277 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=10 tokens_per_ep_rank=[32870, 32768, 32532, 32966, 32686, 32843, 32650, 32829] avg=32768.0000 max=32966 max_rank=3 min=32532 min_rank=2 imbalance_ratio=1.006042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=9 tokens_per_ep_rank=[32558, 32800, 33072, 32851, 32497, 32592, 33058, 32716] avg=32768.0000 max=33072 max_rank=2 min=32497 min_rank=4 imbalance_ratio=1.009277 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=9 tokens_per_ep_rank=[32558, 32800, 33072, 32851, 32497, 32592, 33058, 32716] avg=32768.0000 max=33072 max_rank=2 min=32497 min_rank=4 imbalance_ratio=1.009277 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=10 tokens_per_ep_rank=[32870, 32768, 32532, 32966, 32686, 32843, 32650, 32829] avg=32768.0000 max=32966 max_rank=3 min=32532 min_rank=2 imbalance_ratio=1.006042 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=10 tokens_per_ep_rank=[32870, 32768, 32532, 32966, 32686, 32843, 32650, 32829] avg=32768.0000 max=32966 max_rank=3 min=32532 min_rank=2 imbalance_ratio=1.006042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=10 tokens_per_ep_rank=[32870, 32768, 32532, 32966, 32686, 32843, 32650, 32829] avg=32768.0000 max=32966 max_rank=3 min=32532 min_rank=2 imbalance_ratio=1.006042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=9 tokens_per_ep_rank=[32558, 32800, 33072, 32851, 32497, 32592, 33058, 32716] avg=32768.0000 max=33072 max_rank=2 min=32497 min_rank=4 imbalance_ratio=1.009277 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=10 tokens_per_ep_rank=[32870, 32768, 32532, 32966, 32686, 32843, 32650, 32829] avg=32768.0000 max=32966 max_rank=3 min=32532 min_rank=2 imbalance_ratio=1.006042 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=9 tokens_per_ep_rank=[32558, 32800, 33072, 32851, 32497, 32592, 33058, 32716] avg=32768.0000 max=33072 max_rank=2 min=32497 min_rank=4 imbalance_ratio=1.009277 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=10 tokens_per_ep_rank=[32870, 32768, 32532, 32966, 32686, 32843, 32650, 32829] avg=32768.0000 max=32966 max_rank=3 min=32532 min_rank=2 imbalance_ratio=1.006042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=10 tokens_per_ep_rank=[32870, 32768, 32532, 32966, 32686, 32843, 32650, 32829] avg=32768.0000 max=32966 max_rank=3 min=32532 min_rank=2 imbalance_ratio=1.006042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=10 tokens_per_ep_rank=[32870, 32768, 32532, 32966, 32686, 32843, 32650, 32829] avg=32768.0000 max=32966 max_rank=3 min=32532 min_rank=2 imbalance_ratio=1.006042 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=10 tokens_per_ep_rank=[32870, 32768, 32532, 32966, 32686, 32843, 32650, 32829] avg=32768.0000 max=32966 max_rank=3 min=32532 min_rank=2 imbalance_ratio=1.006042 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=9 tokens_per_ep_rank=[32558, 32800, 33072, 32851, 32497, 32592, 33058, 32716] avg=32768.0000 max=33072 max_rank=2 min=32497 min_rank=4 imbalance_ratio=1.009277 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=10 tokens_per_ep_rank=[32870, 32768, 32532, 32966, 32686, 32843, 32650, 32829] avg=32768.0000 max=32966 max_rank=3 min=32532 min_rank=2 imbalance_ratio=1.006042 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=10 tokens_per_ep_rank=[32870, 32768, 32532, 32966, 32686, 32843, 32650, 32829] avg=32768.0000 max=32966 max_rank=3 min=32532 min_rank=2 imbalance_ratio=1.006042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=10 tokens_per_ep_rank=[32870, 32768, 32532, 32966, 32686, 32843, 32650, 32829] avg=32768.0000 max=32966 max_rank=3 min=32532 min_rank=2 imbalance_ratio=1.006042 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=10 tokens_per_ep_rank=[32870, 32768, 32532, 32966, 32686, 32843, 32650, 32829] avg=32768.0000 max=32966 max_rank=3 min=32532 min_rank=2 imbalance_ratio=1.006042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=10 tokens_per_ep_rank=[32870, 32768, 32532, 32966, 32686, 32843, 32650, 32829] avg=32768.0000 max=32966 max_rank=3 min=32532 min_rank=2 imbalance_ratio=1.006042 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=10 tokens_per_ep_rank=[32870, 32768, 32532, 32966, 32686, 32843, 32650, 32829] avg=32768.0000 max=32966 max_rank=3 min=32532 min_rank=2 imbalance_ratio=1.006042 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=3 records=10 total_expected_recv=327111 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=77312 peak_demand=33072 required_capacity=1.009277 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=3 records=10 total_expected_recv=327681 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76544 peak_demand=32947 required_capacity=1.005463 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=3 records=10 total_expected_recv=327877 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76928 peak_demand=32980 required_capacity=1.006470 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=3 records=10 total_expected_recv=327704 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76928 peak_demand=33006 required_capacity=1.007263 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=3 records=10 total_expected_recv=327635 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76928 peak_demand=32904 required_capacity=1.004150 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=3 records=10 total_expected_recv=327760 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76928 peak_demand=33058 required_capacity=1.008850 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=3 records=10 total_expected_recv=327711 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76032 peak_demand=32930 required_capacity=1.004944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=3 records=10 total_expected_recv=327961 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=77056 peak_demand=33096 required_capacity=1.010010 + [2026-08-26 15:53:12] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 214.5 | throughput per GPU (TFLOP/s/GPU): 575.7 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 1.118410E+01 | loss scale: 1.0 | grad norm: 107.823 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 33664 | min safe capacity: 1.0273 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=0 calls=1 budget=40960 expected_recv=32864 expected_overflow=0 exact_recv_per_rank=[32688, 32859, 32864, 32720, 32860, 32870, 32615, 32668] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=0 calls=1 budget=40960 expected_recv=32870 expected_overflow=0 exact_recv_per_rank=[32688, 32859, 32864, 32720, 32860, 32870, 32615, 32668] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=0 calls=1 budget=40960 expected_recv=32668 expected_overflow=0 exact_recv_per_rank=[32688, 32859, 32864, 32720, 32860, 32870, 32615, 32668] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=0 calls=1 budget=40960 expected_recv=32860 expected_overflow=0 exact_recv_per_rank=[32688, 32859, 32864, 32720, 32860, 32870, 32615, 32668] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=0 calls=1 budget=40960 expected_recv=32859 expected_overflow=0 exact_recv_per_rank=[32688, 32859, 32864, 32720, 32860, 32870, 32615, 32668] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=0 calls=1 budget=40960 expected_recv=32615 expected_overflow=0 exact_recv_per_rank=[32688, 32859, 32864, 32720, 32860, 32870, 32615, 32668] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=0 calls=1 budget=40960 expected_recv=32720 expected_overflow=0 exact_recv_per_rank=[32688, 32859, 32864, 32720, 32860, 32870, 32615, 32668] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=0 calls=1 budget=40960 expected_recv=32864 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=0 calls=1 budget=40960 expected_recv=32870 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=0 calls=1 budget=40960 expected_recv=32668 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=0 calls=1 budget=40960 expected_recv=32688 expected_overflow=0 exact_recv_per_rank=[32688, 32859, 32864, 32720, 32860, 32870, 32615, 32668] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=0 calls=1 budget=40960 expected_recv=32860 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=0 calls=1 budget=40960 expected_recv=32859 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=0 calls=1 budget=40960 expected_recv=32615 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=0 calls=1 budget=40960 expected_recv=32720 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=0 calls=1 budget=40960 expected_recv=32688 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=1 calls=1 budget=40960 expected_recv=32670 expected_overflow=0 exact_recv_per_rank=[32717, 32796, 32569, 32997, 32679, 32670, 32744, 32972] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=1 calls=1 budget=40960 expected_recv=32569 expected_overflow=0 exact_recv_per_rank=[32717, 32796, 32569, 32997, 32679, 32670, 32744, 32972] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=1 calls=1 budget=40960 expected_recv=32972 expected_overflow=0 exact_recv_per_rank=[32717, 32796, 32569, 32997, 32679, 32670, 32744, 32972] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=1 calls=1 budget=40960 expected_recv=32670 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=1 calls=1 budget=40960 expected_recv=32679 expected_overflow=0 exact_recv_per_rank=[32717, 32796, 32569, 32997, 32679, 32670, 32744, 32972] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=1 calls=1 budget=40960 expected_recv=32569 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=1 calls=1 budget=40960 expected_recv=32796 expected_overflow=0 exact_recv_per_rank=[32717, 32796, 32569, 32997, 32679, 32670, 32744, 32972] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=1 calls=1 budget=40960 expected_recv=32972 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=1 calls=1 budget=40960 expected_recv=32997 expected_overflow=0 exact_recv_per_rank=[32717, 32796, 32569, 32997, 32679, 32670, 32744, 32972] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=1 calls=1 budget=40960 expected_recv=32744 expected_overflow=0 exact_recv_per_rank=[32717, 32796, 32569, 32997, 32679, 32670, 32744, 32972] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=1 calls=1 budget=40960 expected_recv=32679 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=1 calls=1 budget=40960 expected_recv=32717 expected_overflow=0 exact_recv_per_rank=[32717, 32796, 32569, 32997, 32679, 32670, 32744, 32972] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=1 calls=1 budget=40960 expected_recv=32796 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=1 calls=1 budget=40960 expected_recv=32997 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=1 calls=1 budget=40960 expected_recv=32744 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=1 calls=1 budget=40960 expected_recv=32717 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=2 calls=1 budget=40960 expected_recv=32675 expected_overflow=0 exact_recv_per_rank=[32675, 33060, 32616, 32822, 32986, 32675, 32765, 32545] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=2 calls=1 budget=40960 expected_recv=32616 expected_overflow=0 exact_recv_per_rank=[32675, 33060, 32616, 32822, 32986, 32675, 32765, 32545] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=2 calls=1 budget=40960 expected_recv=32545 expected_overflow=0 exact_recv_per_rank=[32675, 33060, 32616, 32822, 32986, 32675, 32765, 32545] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=2 calls=1 budget=40960 expected_recv=32675 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=2 calls=1 budget=40960 expected_recv=32986 expected_overflow=0 exact_recv_per_rank=[32675, 33060, 32616, 32822, 32986, 32675, 32765, 32545] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=2 calls=1 budget=40960 expected_recv=32616 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=2 calls=1 budget=40960 expected_recv=32545 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=2 calls=1 budget=40960 expected_recv=33060 expected_overflow=0 exact_recv_per_rank=[32675, 33060, 32616, 32822, 32986, 32675, 32765, 32545] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=2 calls=1 budget=40960 expected_recv=32822 expected_overflow=0 exact_recv_per_rank=[32675, 33060, 32616, 32822, 32986, 32675, 32765, 32545] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=2 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[32675, 33060, 32616, 32822, 32986, 32675, 32765, 32545] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=2 calls=1 budget=40960 expected_recv=32986 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=2 calls=1 budget=40960 expected_recv=32675 expected_overflow=0 exact_recv_per_rank=[32675, 33060, 32616, 32822, 32986, 32675, 32765, 32545] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=2 calls=1 budget=40960 expected_recv=33060 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=2 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=2 calls=1 budget=40960 expected_recv=32822 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=2 calls=1 budget=40960 expected_recv=32675 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=3 calls=1 budget=40960 expected_recv=32560 expected_overflow=0 exact_recv_per_rank=[32964, 32913, 33096, 32509, 32745, 32560, 32508, 32849] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=3 calls=1 budget=40960 expected_recv=32560 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32964, 32913, 33096, 32509, 32745, 32560, 32508, 32849] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=3 calls=1 budget=40960 expected_recv=32849 expected_overflow=0 exact_recv_per_rank=[32964, 32913, 33096, 32509, 32745, 32560, 32508, 32849] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=3 calls=1 budget=40960 expected_recv=32745 expected_overflow=0 exact_recv_per_rank=[32964, 32913, 33096, 32509, 32745, 32560, 32508, 32849] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=3 calls=1 budget=40960 expected_recv=32849 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=3 calls=1 budget=40960 expected_recv=32745 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=3 calls=1 budget=40960 expected_recv=32913 expected_overflow=0 exact_recv_per_rank=[32964, 32913, 33096, 32509, 32745, 32560, 32508, 32849] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=3 calls=1 budget=40960 expected_recv=32509 expected_overflow=0 exact_recv_per_rank=[32964, 32913, 33096, 32509, 32745, 32560, 32508, 32849] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=3 calls=1 budget=40960 expected_recv=32508 expected_overflow=0 exact_recv_per_rank=[32964, 32913, 33096, 32509, 32745, 32560, 32508, 32849] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=3 calls=1 budget=40960 expected_recv=32964 expected_overflow=0 exact_recv_per_rank=[32964, 32913, 33096, 32509, 32745, 32560, 32508, 32849] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=3 calls=1 budget=40960 expected_recv=32508 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=3 calls=1 budget=40960 expected_recv=32913 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=3 calls=1 budget=40960 expected_recv=32509 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=3 calls=1 budget=40960 expected_recv=32964 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=4 calls=1 budget=40960 expected_recv=32742 expected_overflow=0 exact_recv_per_rank=[32502, 32690, 32933, 32932, 32763, 32742, 32723, 32859] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=4 calls=1 budget=40960 expected_recv=32742 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=4 calls=1 budget=40960 expected_recv=32933 expected_overflow=0 exact_recv_per_rank=[32502, 32690, 32933, 32932, 32763, 32742, 32723, 32859] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=4 calls=1 budget=40960 expected_recv=32933 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=4 calls=1 budget=40960 expected_recv=32859 expected_overflow=0 exact_recv_per_rank=[32502, 32690, 32933, 32932, 32763, 32742, 32723, 32859] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=4 calls=1 budget=40960 expected_recv=32763 expected_overflow=0 exact_recv_per_rank=[32502, 32690, 32933, 32932, 32763, 32742, 32723, 32859] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=4 calls=1 budget=40960 expected_recv=32859 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=4 calls=1 budget=40960 expected_recv=32763 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=4 calls=1 budget=40960 expected_recv=32723 expected_overflow=0 exact_recv_per_rank=[32502, 32690, 32933, 32932, 32763, 32742, 32723, 32859] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=4 calls=1 budget=40960 expected_recv=32690 expected_overflow=0 exact_recv_per_rank=[32502, 32690, 32933, 32932, 32763, 32742, 32723, 32859] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=4 calls=1 budget=40960 expected_recv=32932 expected_overflow=0 exact_recv_per_rank=[32502, 32690, 32933, 32932, 32763, 32742, 32723, 32859] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=4 calls=1 budget=40960 expected_recv=32502 expected_overflow=0 exact_recv_per_rank=[32502, 32690, 32933, 32932, 32763, 32742, 32723, 32859] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=4 calls=1 budget=40960 expected_recv=32723 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=5 calls=1 budget=40960 expected_recv=32782 expected_overflow=0 exact_recv_per_rank=[32782, 32883, 32542, 32679, 32920, 32782, 32797, 32759] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=4 calls=1 budget=40960 expected_recv=32690 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=4 calls=1 budget=40960 expected_recv=32932 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=4 calls=1 budget=40960 expected_recv=32502 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=5 calls=1 budget=40960 expected_recv=32782 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=5 calls=1 budget=40960 expected_recv=32542 expected_overflow=0 exact_recv_per_rank=[32782, 32883, 32542, 32679, 32920, 32782, 32797, 32759] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=5 calls=1 budget=40960 expected_recv=32542 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=5 calls=1 budget=40960 expected_recv=32759 expected_overflow=0 exact_recv_per_rank=[32782, 32883, 32542, 32679, 32920, 32782, 32797, 32759] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=5 calls=1 budget=40960 expected_recv=32920 expected_overflow=0 exact_recv_per_rank=[32782, 32883, 32542, 32679, 32920, 32782, 32797, 32759] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=5 calls=1 budget=40960 expected_recv=32759 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=5 calls=1 budget=40960 expected_recv=32797 expected_overflow=0 exact_recv_per_rank=[32782, 32883, 32542, 32679, 32920, 32782, 32797, 32759] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=5 calls=1 budget=40960 expected_recv=32920 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=5 calls=1 budget=40960 expected_recv=32679 expected_overflow=0 exact_recv_per_rank=[32782, 32883, 32542, 32679, 32920, 32782, 32797, 32759] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=5 calls=1 budget=40960 expected_recv=32883 expected_overflow=0 exact_recv_per_rank=[32782, 32883, 32542, 32679, 32920, 32782, 32797, 32759] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=6 calls=1 budget=40960 expected_recv=32793 expected_overflow=0 exact_recv_per_rank=[32971, 33018, 32966, 32453, 32518, 32793, 32634, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=5 calls=1 budget=40960 expected_recv=32782 expected_overflow=0 exact_recv_per_rank=[32782, 32883, 32542, 32679, 32920, 32782, 32797, 32759] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=5 calls=1 budget=40960 expected_recv=32797 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=5 calls=1 budget=40960 expected_recv=32679 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=5 calls=1 budget=40960 expected_recv=32883 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=6 calls=1 budget=40960 expected_recv=32793 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=5 calls=1 budget=40960 expected_recv=32782 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=6 calls=1 budget=40960 expected_recv=32966 expected_overflow=0 exact_recv_per_rank=[32971, 33018, 32966, 32453, 32518, 32793, 32634, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=6 calls=1 budget=40960 expected_recv=32966 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=6 calls=1 budget=40960 expected_recv=32791 expected_overflow=0 exact_recv_per_rank=[32971, 33018, 32966, 32453, 32518, 32793, 32634, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=6 calls=1 budget=40960 expected_recv=32791 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=6 calls=1 budget=40960 expected_recv=32518 expected_overflow=0 exact_recv_per_rank=[32971, 33018, 32966, 32453, 32518, 32793, 32634, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=6 calls=1 budget=40960 expected_recv=32518 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=6 calls=1 budget=40960 expected_recv=32634 expected_overflow=0 exact_recv_per_rank=[32971, 33018, 32966, 32453, 32518, 32793, 32634, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=6 calls=1 budget=40960 expected_recv=32453 expected_overflow=0 exact_recv_per_rank=[32971, 33018, 32966, 32453, 32518, 32793, 32634, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=7 calls=1 budget=40960 expected_recv=32911 expected_overflow=0 exact_recv_per_rank=[32954, 32697, 32700, 32666, 32757, 32911, 32753, 32706] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=6 calls=1 budget=40960 expected_recv=33018 expected_overflow=0 exact_recv_per_rank=[32971, 33018, 32966, 32453, 32518, 32793, 32634, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=6 calls=1 budget=40960 expected_recv=32634 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=6 calls=1 budget=40960 expected_recv=32971 expected_overflow=0 exact_recv_per_rank=[32971, 33018, 32966, 32453, 32518, 32793, 32634, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=6 calls=1 budget=40960 expected_recv=32453 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=7 calls=1 budget=40960 expected_recv=32911 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=6 calls=1 budget=40960 expected_recv=33018 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=7 calls=1 budget=40960 expected_recv=32700 expected_overflow=0 exact_recv_per_rank=[32954, 32697, 32700, 32666, 32757, 32911, 32753, 32706] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=6 calls=1 budget=40960 expected_recv=32971 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=7 calls=1 budget=40960 expected_recv=32706 expected_overflow=0 exact_recv_per_rank=[32954, 32697, 32700, 32666, 32757, 32911, 32753, 32706] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=7 calls=1 budget=40960 expected_recv=32700 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=7 calls=1 budget=40960 expected_recv=32757 expected_overflow=0 exact_recv_per_rank=[32954, 32697, 32700, 32666, 32757, 32911, 32753, 32706] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=7 calls=1 budget=40960 expected_recv=32706 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=7 calls=1 budget=40960 expected_recv=32757 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=7 calls=1 budget=40960 expected_recv=32753 expected_overflow=0 exact_recv_per_rank=[32954, 32697, 32700, 32666, 32757, 32911, 32753, 32706] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=7 calls=1 budget=40960 expected_recv=32666 expected_overflow=0 exact_recv_per_rank=[32954, 32697, 32700, 32666, 32757, 32911, 32753, 32706] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32956 expected_overflow=0 exact_recv_per_rank=[32853, 32918, 32465, 32763, 32776, 32956, 32616, 32797] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=7 calls=1 budget=40960 expected_recv=32697 expected_overflow=0 exact_recv_per_rank=[32954, 32697, 32700, 32666, 32757, 32911, 32753, 32706] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=7 calls=1 budget=40960 expected_recv=32753 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=7 calls=1 budget=40960 expected_recv=32666 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=7 calls=1 budget=40960 expected_recv=32954 expected_overflow=0 exact_recv_per_rank=[32954, 32697, 32700, 32666, 32757, 32911, 32753, 32706] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32465 expected_overflow=0 exact_recv_per_rank=[32853, 32918, 32465, 32763, 32776, 32956, 32616, 32797] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32956 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=7 calls=1 budget=40960 expected_recv=32697 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32797 expected_overflow=0 exact_recv_per_rank=[32853, 32918, 32465, 32763, 32776, 32956, 32616, 32797] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32465 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32776 expected_overflow=0 exact_recv_per_rank=[32853, 32918, 32465, 32763, 32776, 32956, 32616, 32797] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=7 calls=1 budget=40960 expected_recv=32954 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32797 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32776 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32616 expected_overflow=0 exact_recv_per_rank=[32853, 32918, 32465, 32763, 32776, 32956, 32616, 32797] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32777 expected_overflow=0 exact_recv_per_rank=[32955, 32898, 32306, 32813, 32759, 32777, 32800, 32836] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32763 expected_overflow=0 exact_recv_per_rank=[32853, 32918, 32465, 32763, 32776, 32956, 32616, 32797] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32616 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32918 expected_overflow=0 exact_recv_per_rank=[32853, 32918, 32465, 32763, 32776, 32956, 32616, 32797] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32306 expected_overflow=0 exact_recv_per_rank=[32955, 32898, 32306, 32813, 32759, 32777, 32800, 32836] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32777 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32763 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32918 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32306 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32836 expected_overflow=0 exact_recv_per_rank=[32955, 32898, 32306, 32813, 32759, 32777, 32800, 32836] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32853 expected_overflow=0 exact_recv_per_rank=[32853, 32918, 32465, 32763, 32776, 32956, 32616, 32797] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32759 expected_overflow=0 exact_recv_per_rank=[32955, 32898, 32306, 32813, 32759, 32777, 32800, 32836] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32836 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32853 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32759 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32800 expected_overflow=0 exact_recv_per_rank=[32955, 32898, 32306, 32813, 32759, 32777, 32800, 32836] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=1 tokens_per_ep_rank=[32688, 32859, 32864, 32720, 32860, 32870, 32615, 32668] avg=32768.0000 max=32870 max_rank=5 min=32615 min_rank=6 imbalance_ratio=1.003113 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32813 expected_overflow=0 exact_recv_per_rank=[32955, 32898, 32306, 32813, 32759, 32777, 32800, 32836] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32800 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32898 expected_overflow=0 exact_recv_per_rank=[32955, 32898, 32306, 32813, 32759, 32777, 32800, 32836] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=1 tokens_per_ep_rank=[32688, 32859, 32864, 32720, 32860, 32870, 32615, 32668] avg=32768.0000 max=32870 max_rank=5 min=32615 min_rank=6 imbalance_ratio=1.003113 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32813 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32898 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32955 expected_overflow=0 exact_recv_per_rank=[32955, 32898, 32306, 32813, 32759, 32777, 32800, 32836] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=1 tokens_per_ep_rank=[32688, 32859, 32864, 32720, 32860, 32870, 32615, 32668] avg=32768.0000 max=32870 max_rank=5 min=32615 min_rank=6 imbalance_ratio=1.003113 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=1 tokens_per_ep_rank=[32688, 32859, 32864, 32720, 32860, 32870, 32615, 32668] avg=32768.0000 max=32870 max_rank=5 min=32615 min_rank=6 imbalance_ratio=1.003113 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=1 tokens_per_ep_rank=[32688, 32859, 32864, 32720, 32860, 32870, 32615, 32668] avg=32768.0000 max=32870 max_rank=5 min=32615 min_rank=6 imbalance_ratio=1.003113 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32955 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=1 tokens_per_ep_rank=[32688, 32859, 32864, 32720, 32860, 32870, 32615, 32668] avg=32768.0000 max=32870 max_rank=5 min=32615 min_rank=6 imbalance_ratio=1.003113 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=1 tokens_per_ep_rank=[32688, 32859, 32864, 32720, 32860, 32870, 32615, 32668] avg=32768.0000 max=32870 max_rank=5 min=32615 min_rank=6 imbalance_ratio=1.003113 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=1 tokens_per_ep_rank=[32688, 32859, 32864, 32720, 32860, 32870, 32615, 32668] avg=32768.0000 max=32870 max_rank=5 min=32615 min_rank=6 imbalance_ratio=1.003113 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=1 tokens_per_ep_rank=[32688, 32859, 32864, 32720, 32860, 32870, 32615, 32668] avg=32768.0000 max=32870 max_rank=5 min=32615 min_rank=6 imbalance_ratio=1.003113 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=1 tokens_per_ep_rank=[32688, 32859, 32864, 32720, 32860, 32870, 32615, 32668] avg=32768.0000 max=32870 max_rank=5 min=32615 min_rank=6 imbalance_ratio=1.003113 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=2 tokens_per_ep_rank=[32717, 32796, 32569, 32997, 32679, 32670, 32744, 32972] avg=32768.0000 max=32997 max_rank=3 min=32569 min_rank=2 imbalance_ratio=1.006989 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=1 tokens_per_ep_rank=[32688, 32859, 32864, 32720, 32860, 32870, 32615, 32668] avg=32768.0000 max=32870 max_rank=5 min=32615 min_rank=6 imbalance_ratio=1.003113 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=2 tokens_per_ep_rank=[32717, 32796, 32569, 32997, 32679, 32670, 32744, 32972] avg=32768.0000 max=32997 max_rank=3 min=32569 min_rank=2 imbalance_ratio=1.006989 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=1 tokens_per_ep_rank=[32688, 32859, 32864, 32720, 32860, 32870, 32615, 32668] avg=32768.0000 max=32870 max_rank=5 min=32615 min_rank=6 imbalance_ratio=1.003113 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=1 tokens_per_ep_rank=[32688, 32859, 32864, 32720, 32860, 32870, 32615, 32668] avg=32768.0000 max=32870 max_rank=5 min=32615 min_rank=6 imbalance_ratio=1.003113 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=2 tokens_per_ep_rank=[32717, 32796, 32569, 32997, 32679, 32670, 32744, 32972] avg=32768.0000 max=32997 max_rank=3 min=32569 min_rank=2 imbalance_ratio=1.006989 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=2 tokens_per_ep_rank=[32717, 32796, 32569, 32997, 32679, 32670, 32744, 32972] avg=32768.0000 max=32997 max_rank=3 min=32569 min_rank=2 imbalance_ratio=1.006989 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=1 tokens_per_ep_rank=[32688, 32859, 32864, 32720, 32860, 32870, 32615, 32668] avg=32768.0000 max=32870 max_rank=5 min=32615 min_rank=6 imbalance_ratio=1.003113 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=2 tokens_per_ep_rank=[32717, 32796, 32569, 32997, 32679, 32670, 32744, 32972] avg=32768.0000 max=32997 max_rank=3 min=32569 min_rank=2 imbalance_ratio=1.006989 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=1 tokens_per_ep_rank=[32688, 32859, 32864, 32720, 32860, 32870, 32615, 32668] avg=32768.0000 max=32870 max_rank=5 min=32615 min_rank=6 imbalance_ratio=1.003113 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=2 tokens_per_ep_rank=[32717, 32796, 32569, 32997, 32679, 32670, 32744, 32972] avg=32768.0000 max=32997 max_rank=3 min=32569 min_rank=2 imbalance_ratio=1.006989 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=1 tokens_per_ep_rank=[32688, 32859, 32864, 32720, 32860, 32870, 32615, 32668] avg=32768.0000 max=32870 max_rank=5 min=32615 min_rank=6 imbalance_ratio=1.003113 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=2 tokens_per_ep_rank=[32717, 32796, 32569, 32997, 32679, 32670, 32744, 32972] avg=32768.0000 max=32997 max_rank=3 min=32569 min_rank=2 imbalance_ratio=1.006989 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=2 tokens_per_ep_rank=[32717, 32796, 32569, 32997, 32679, 32670, 32744, 32972] avg=32768.0000 max=32997 max_rank=3 min=32569 min_rank=2 imbalance_ratio=1.006989 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=2 tokens_per_ep_rank=[32717, 32796, 32569, 32997, 32679, 32670, 32744, 32972] avg=32768.0000 max=32997 max_rank=3 min=32569 min_rank=2 imbalance_ratio=1.006989 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=3 tokens_per_ep_rank=[32675, 33060, 32616, 32822, 32986, 32675, 32765, 32545] avg=32768.0000 max=33060 max_rank=1 min=32545 min_rank=7 imbalance_ratio=1.008911 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=2 tokens_per_ep_rank=[32717, 32796, 32569, 32997, 32679, 32670, 32744, 32972] avg=32768.0000 max=32997 max_rank=3 min=32569 min_rank=2 imbalance_ratio=1.006989 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=3 tokens_per_ep_rank=[32675, 33060, 32616, 32822, 32986, 32675, 32765, 32545] avg=32768.0000 max=33060 max_rank=1 min=32545 min_rank=7 imbalance_ratio=1.008911 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=2 tokens_per_ep_rank=[32717, 32796, 32569, 32997, 32679, 32670, 32744, 32972] avg=32768.0000 max=32997 max_rank=3 min=32569 min_rank=2 imbalance_ratio=1.006989 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=2 tokens_per_ep_rank=[32717, 32796, 32569, 32997, 32679, 32670, 32744, 32972] avg=32768.0000 max=32997 max_rank=3 min=32569 min_rank=2 imbalance_ratio=1.006989 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=3 tokens_per_ep_rank=[32675, 33060, 32616, 32822, 32986, 32675, 32765, 32545] avg=32768.0000 max=33060 max_rank=1 min=32545 min_rank=7 imbalance_ratio=1.008911 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=3 tokens_per_ep_rank=[32675, 33060, 32616, 32822, 32986, 32675, 32765, 32545] avg=32768.0000 max=33060 max_rank=1 min=32545 min_rank=7 imbalance_ratio=1.008911 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=2 tokens_per_ep_rank=[32717, 32796, 32569, 32997, 32679, 32670, 32744, 32972] avg=32768.0000 max=32997 max_rank=3 min=32569 min_rank=2 imbalance_ratio=1.006989 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=3 tokens_per_ep_rank=[32675, 33060, 32616, 32822, 32986, 32675, 32765, 32545] avg=32768.0000 max=33060 max_rank=1 min=32545 min_rank=7 imbalance_ratio=1.008911 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=3 tokens_per_ep_rank=[32675, 33060, 32616, 32822, 32986, 32675, 32765, 32545] avg=32768.0000 max=33060 max_rank=1 min=32545 min_rank=7 imbalance_ratio=1.008911 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=2 tokens_per_ep_rank=[32717, 32796, 32569, 32997, 32679, 32670, 32744, 32972] avg=32768.0000 max=32997 max_rank=3 min=32569 min_rank=2 imbalance_ratio=1.006989 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=2 tokens_per_ep_rank=[32717, 32796, 32569, 32997, 32679, 32670, 32744, 32972] avg=32768.0000 max=32997 max_rank=3 min=32569 min_rank=2 imbalance_ratio=1.006989 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=3 tokens_per_ep_rank=[32675, 33060, 32616, 32822, 32986, 32675, 32765, 32545] avg=32768.0000 max=33060 max_rank=1 min=32545 min_rank=7 imbalance_ratio=1.008911 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=2 tokens_per_ep_rank=[32717, 32796, 32569, 32997, 32679, 32670, 32744, 32972] avg=32768.0000 max=32997 max_rank=3 min=32569 min_rank=2 imbalance_ratio=1.006989 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=3 tokens_per_ep_rank=[32675, 33060, 32616, 32822, 32986, 32675, 32765, 32545] avg=32768.0000 max=33060 max_rank=1 min=32545 min_rank=7 imbalance_ratio=1.008911 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=3 tokens_per_ep_rank=[32675, 33060, 32616, 32822, 32986, 32675, 32765, 32545] avg=32768.0000 max=33060 max_rank=1 min=32545 min_rank=7 imbalance_ratio=1.008911 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=4 tokens_per_ep_rank=[32964, 32913, 33156, 32509, 32745, 32560, 32448, 32849] avg=32768.0000 max=33156 max_rank=2 min=32448 min_rank=6 imbalance_ratio=1.011841 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=4 tokens_per_ep_rank=[32964, 32913, 33156, 32509, 32745, 32560, 32448, 32849] avg=32768.0000 max=33156 max_rank=2 min=32448 min_rank=6 imbalance_ratio=1.011841 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=3 tokens_per_ep_rank=[32675, 33060, 32616, 32822, 32986, 32675, 32765, 32545] avg=32768.0000 max=33060 max_rank=1 min=32545 min_rank=7 imbalance_ratio=1.008911 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=3 tokens_per_ep_rank=[32675, 33060, 32616, 32822, 32986, 32675, 32765, 32545] avg=32768.0000 max=33060 max_rank=1 min=32545 min_rank=7 imbalance_ratio=1.008911 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=4 tokens_per_ep_rank=[32964, 32913, 33096, 32509, 32745, 32560, 32508, 32849] avg=32768.0000 max=33096 max_rank=2 min=32508 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=60 max_load_reduction_pct=0.001810 moved_to_replicas=60 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=3 tokens_per_ep_rank=[32675, 33060, 32616, 32822, 32986, 32675, 32765, 32545] avg=32768.0000 max=33060 max_rank=1 min=32545 min_rank=7 imbalance_ratio=1.008911 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=4 tokens_per_ep_rank=[32964, 32913, 33156, 32509, 32745, 32560, 32448, 32849] avg=32768.0000 max=33156 max_rank=2 min=32448 min_rank=6 imbalance_ratio=1.011841 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=4 tokens_per_ep_rank=[32964, 32913, 33156, 32509, 32745, 32560, 32448, 32849] avg=32768.0000 max=33156 max_rank=2 min=32448 min_rank=6 imbalance_ratio=1.011841 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=3 tokens_per_ep_rank=[32675, 33060, 32616, 32822, 32986, 32675, 32765, 32545] avg=32768.0000 max=33060 max_rank=1 min=32545 min_rank=7 imbalance_ratio=1.008911 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=4 tokens_per_ep_rank=[32964, 32913, 33096, 32509, 32745, 32560, 32508, 32849] avg=32768.0000 max=33096 max_rank=2 min=32508 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=60 max_load_reduction_pct=0.001810 moved_to_replicas=60 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=3 tokens_per_ep_rank=[32675, 33060, 32616, 32822, 32986, 32675, 32765, 32545] avg=32768.0000 max=33060 max_rank=1 min=32545 min_rank=7 imbalance_ratio=1.008911 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=3 tokens_per_ep_rank=[32675, 33060, 32616, 32822, 32986, 32675, 32765, 32545] avg=32768.0000 max=33060 max_rank=1 min=32545 min_rank=7 imbalance_ratio=1.008911 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=4 tokens_per_ep_rank=[32964, 32913, 33096, 32509, 32745, 32560, 32508, 32849] avg=32768.0000 max=33096 max_rank=2 min=32508 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=60 max_load_reduction_pct=0.001810 moved_to_replicas=60 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=4 tokens_per_ep_rank=[32964, 32913, 33096, 32509, 32745, 32560, 32508, 32849] avg=32768.0000 max=33096 max_rank=2 min=32508 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=60 max_load_reduction_pct=0.001810 moved_to_replicas=60 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=5 tokens_per_ep_rank=[32502, 32690, 32933, 32932, 32763, 32742, 32723, 32859] avg=32768.0000 max=32933 max_rank=2 min=32502 min_rank=0 imbalance_ratio=1.005035 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=3 tokens_per_ep_rank=[32675, 33060, 32616, 32822, 32986, 32675, 32765, 32545] avg=32768.0000 max=33060 max_rank=1 min=32545 min_rank=7 imbalance_ratio=1.008911 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=4 tokens_per_ep_rank=[32964, 32913, 33156, 32509, 32745, 32560, 32448, 32849] avg=32768.0000 max=33156 max_rank=2 min=32448 min_rank=6 imbalance_ratio=1.011841 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=5 tokens_per_ep_rank=[32502, 32690, 32933, 32932, 32763, 32742, 32723, 32859] avg=32768.0000 max=32933 max_rank=2 min=32502 min_rank=0 imbalance_ratio=1.005035 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=4 tokens_per_ep_rank=[32964, 32913, 33156, 32509, 32745, 32560, 32448, 32849] avg=32768.0000 max=33156 max_rank=2 min=32448 min_rank=6 imbalance_ratio=1.011841 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=4 tokens_per_ep_rank=[32964, 32913, 33156, 32509, 32745, 32560, 32448, 32849] avg=32768.0000 max=33156 max_rank=2 min=32448 min_rank=6 imbalance_ratio=1.011841 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=5 tokens_per_ep_rank=[32502, 32690, 32933, 32932, 32763, 32742, 32723, 32859] avg=32768.0000 max=32933 max_rank=2 min=32502 min_rank=0 imbalance_ratio=1.005035 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=5 tokens_per_ep_rank=[32502, 32690, 32933, 32932, 32763, 32742, 32723, 32859] avg=32768.0000 max=32933 max_rank=2 min=32502 min_rank=0 imbalance_ratio=1.005035 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=4 tokens_per_ep_rank=[32964, 32913, 33096, 32509, 32745, 32560, 32508, 32849] avg=32768.0000 max=33096 max_rank=2 min=32508 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=60 max_load_reduction_pct=0.001810 moved_to_replicas=60 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=5 tokens_per_ep_rank=[32502, 32690, 32933, 32932, 32763, 32742, 32723, 32859] avg=32768.0000 max=32933 max_rank=2 min=32502 min_rank=0 imbalance_ratio=1.005035 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=4 tokens_per_ep_rank=[32964, 32913, 33156, 32509, 32745, 32560, 32448, 32849] avg=32768.0000 max=33156 max_rank=2 min=32448 min_rank=6 imbalance_ratio=1.011841 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=5 tokens_per_ep_rank=[32502, 32690, 32933, 32932, 32763, 32742, 32723, 32859] avg=32768.0000 max=32933 max_rank=2 min=32502 min_rank=0 imbalance_ratio=1.005035 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=4 tokens_per_ep_rank=[32964, 32913, 33096, 32509, 32745, 32560, 32508, 32849] avg=32768.0000 max=33096 max_rank=2 min=32508 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=60 max_load_reduction_pct=0.001810 moved_to_replicas=60 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=5 tokens_per_ep_rank=[32502, 32690, 32933, 32932, 32763, 32742, 32723, 32859] avg=32768.0000 max=32933 max_rank=2 min=32502 min_rank=0 imbalance_ratio=1.005035 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=4 tokens_per_ep_rank=[32964, 32913, 33096, 32509, 32745, 32560, 32508, 32849] avg=32768.0000 max=33096 max_rank=2 min=32508 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=60 max_load_reduction_pct=0.001810 moved_to_replicas=60 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=5 tokens_per_ep_rank=[32502, 32690, 32933, 32932, 32763, 32742, 32723, 32859] avg=32768.0000 max=32933 max_rank=2 min=32502 min_rank=0 imbalance_ratio=1.005035 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=6 tokens_per_ep_rank=[32782, 32883, 32542, 32679, 32920, 32782, 32797, 32759] avg=32768.0000 max=32920 max_rank=4 min=32542 min_rank=2 imbalance_ratio=1.004639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=5 tokens_per_ep_rank=[32502, 32690, 32933, 32932, 32763, 32742, 32723, 32859] avg=32768.0000 max=32933 max_rank=2 min=32502 min_rank=0 imbalance_ratio=1.005035 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=4 tokens_per_ep_rank=[32964, 32913, 33096, 32509, 32745, 32560, 32508, 32849] avg=32768.0000 max=33096 max_rank=2 min=32508 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=60 max_load_reduction_pct=0.001810 moved_to_replicas=60 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=6 tokens_per_ep_rank=[32782, 32883, 32542, 32679, 32920, 32782, 32797, 32759] avg=32768.0000 max=32920 max_rank=4 min=32542 min_rank=2 imbalance_ratio=1.004639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=6 tokens_per_ep_rank=[32782, 32883, 32542, 32679, 32920, 32782, 32797, 32759] avg=32768.0000 max=32920 max_rank=4 min=32542 min_rank=2 imbalance_ratio=1.004639 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=5 tokens_per_ep_rank=[32502, 32690, 32933, 32932, 32763, 32742, 32723, 32859] avg=32768.0000 max=32933 max_rank=2 min=32502 min_rank=0 imbalance_ratio=1.005035 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=5 tokens_per_ep_rank=[32502, 32690, 32933, 32932, 32763, 32742, 32723, 32859] avg=32768.0000 max=32933 max_rank=2 min=32502 min_rank=0 imbalance_ratio=1.005035 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=6 tokens_per_ep_rank=[32782, 32883, 32542, 32679, 32920, 32782, 32797, 32759] avg=32768.0000 max=32920 max_rank=4 min=32542 min_rank=2 imbalance_ratio=1.004639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=5 tokens_per_ep_rank=[32502, 32690, 32933, 32932, 32763, 32742, 32723, 32859] avg=32768.0000 max=32933 max_rank=2 min=32502 min_rank=0 imbalance_ratio=1.005035 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=6 tokens_per_ep_rank=[32782, 32883, 32542, 32679, 32920, 32782, 32797, 32759] avg=32768.0000 max=32920 max_rank=4 min=32542 min_rank=2 imbalance_ratio=1.004639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=6 tokens_per_ep_rank=[32782, 32883, 32542, 32679, 32920, 32782, 32797, 32759] avg=32768.0000 max=32920 max_rank=4 min=32542 min_rank=2 imbalance_ratio=1.004639 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=5 tokens_per_ep_rank=[32502, 32690, 32933, 32932, 32763, 32742, 32723, 32859] avg=32768.0000 max=32933 max_rank=2 min=32502 min_rank=0 imbalance_ratio=1.005035 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=5 tokens_per_ep_rank=[32502, 32690, 32933, 32932, 32763, 32742, 32723, 32859] avg=32768.0000 max=32933 max_rank=2 min=32502 min_rank=0 imbalance_ratio=1.005035 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=5 tokens_per_ep_rank=[32502, 32690, 32933, 32932, 32763, 32742, 32723, 32859] avg=32768.0000 max=32933 max_rank=2 min=32502 min_rank=0 imbalance_ratio=1.005035 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=6 tokens_per_ep_rank=[32782, 32883, 32542, 32679, 32920, 32782, 32797, 32759] avg=32768.0000 max=32920 max_rank=4 min=32542 min_rank=2 imbalance_ratio=1.004639 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=6 tokens_per_ep_rank=[32782, 32883, 32542, 32679, 32920, 32782, 32797, 32759] avg=32768.0000 max=32920 max_rank=4 min=32542 min_rank=2 imbalance_ratio=1.004639 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=7 tokens_per_ep_rank=[32971, 33018, 32966, 32453, 32518, 32793, 32634, 32791] avg=32768.0000 max=33018 max_rank=1 min=32453 min_rank=3 imbalance_ratio=1.007629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=6 tokens_per_ep_rank=[32782, 32883, 32542, 32679, 32920, 32782, 32797, 32759] avg=32768.0000 max=32920 max_rank=4 min=32542 min_rank=2 imbalance_ratio=1.004639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=5 tokens_per_ep_rank=[32502, 32690, 32933, 32932, 32763, 32742, 32723, 32859] avg=32768.0000 max=32933 max_rank=2 min=32502 min_rank=0 imbalance_ratio=1.005035 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=7 tokens_per_ep_rank=[32971, 33018, 32966, 32453, 32518, 32793, 32634, 32791] avg=32768.0000 max=33018 max_rank=1 min=32453 min_rank=3 imbalance_ratio=1.007629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=7 tokens_per_ep_rank=[32971, 33018, 32966, 32453, 32518, 32793, 32634, 32791] avg=32768.0000 max=33018 max_rank=1 min=32453 min_rank=3 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=6 tokens_per_ep_rank=[32782, 32883, 32542, 32679, 32920, 32782, 32797, 32759] avg=32768.0000 max=32920 max_rank=4 min=32542 min_rank=2 imbalance_ratio=1.004639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=6 tokens_per_ep_rank=[32782, 32883, 32542, 32679, 32920, 32782, 32797, 32759] avg=32768.0000 max=32920 max_rank=4 min=32542 min_rank=2 imbalance_ratio=1.004639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=7 tokens_per_ep_rank=[32971, 33018, 32966, 32453, 32518, 32793, 32634, 32791] avg=32768.0000 max=33018 max_rank=1 min=32453 min_rank=3 imbalance_ratio=1.007629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=7 tokens_per_ep_rank=[32971, 33018, 32966, 32453, 32518, 32793, 32634, 32791] avg=32768.0000 max=33018 max_rank=1 min=32453 min_rank=3 imbalance_ratio=1.007629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=6 tokens_per_ep_rank=[32782, 32883, 32542, 32679, 32920, 32782, 32797, 32759] avg=32768.0000 max=32920 max_rank=4 min=32542 min_rank=2 imbalance_ratio=1.004639 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=7 tokens_per_ep_rank=[32971, 33018, 32966, 32453, 32518, 32793, 32634, 32791] avg=32768.0000 max=33018 max_rank=1 min=32453 min_rank=3 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=6 tokens_per_ep_rank=[32782, 32883, 32542, 32679, 32920, 32782, 32797, 32759] avg=32768.0000 max=32920 max_rank=4 min=32542 min_rank=2 imbalance_ratio=1.004639 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=6 tokens_per_ep_rank=[32782, 32883, 32542, 32679, 32920, 32782, 32797, 32759] avg=32768.0000 max=32920 max_rank=4 min=32542 min_rank=2 imbalance_ratio=1.004639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=6 tokens_per_ep_rank=[32782, 32883, 32542, 32679, 32920, 32782, 32797, 32759] avg=32768.0000 max=32920 max_rank=4 min=32542 min_rank=2 imbalance_ratio=1.004639 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=7 tokens_per_ep_rank=[32971, 33018, 32966, 32453, 32518, 32793, 32634, 32791] avg=32768.0000 max=33018 max_rank=1 min=32453 min_rank=3 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=7 tokens_per_ep_rank=[32971, 33018, 32966, 32453, 32518, 32793, 32634, 32791] avg=32768.0000 max=33018 max_rank=1 min=32453 min_rank=3 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=8 tokens_per_ep_rank=[32954, 32697, 32700, 32666, 32757, 32911, 32753, 32706] avg=32768.0000 max=32954 max_rank=0 min=32666 min_rank=3 imbalance_ratio=1.005676 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=7 tokens_per_ep_rank=[32971, 33018, 32966, 32453, 32518, 32793, 32634, 32791] avg=32768.0000 max=33018 max_rank=1 min=32453 min_rank=3 imbalance_ratio=1.007629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=6 tokens_per_ep_rank=[32782, 32883, 32542, 32679, 32920, 32782, 32797, 32759] avg=32768.0000 max=32920 max_rank=4 min=32542 min_rank=2 imbalance_ratio=1.004639 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=8 tokens_per_ep_rank=[32954, 32697, 32700, 32666, 32757, 32911, 32753, 32706] avg=32768.0000 max=32954 max_rank=0 min=32666 min_rank=3 imbalance_ratio=1.005676 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=8 tokens_per_ep_rank=[32954, 32697, 32700, 32666, 32757, 32911, 32753, 32706] avg=32768.0000 max=32954 max_rank=0 min=32666 min_rank=3 imbalance_ratio=1.005676 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=7 tokens_per_ep_rank=[32971, 33018, 32966, 32453, 32518, 32793, 32634, 32791] avg=32768.0000 max=33018 max_rank=1 min=32453 min_rank=3 imbalance_ratio=1.007629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=7 tokens_per_ep_rank=[32971, 33018, 32966, 32453, 32518, 32793, 32634, 32791] avg=32768.0000 max=33018 max_rank=1 min=32453 min_rank=3 imbalance_ratio=1.007629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=8 tokens_per_ep_rank=[32954, 32697, 32700, 32666, 32757, 32911, 32753, 32706] avg=32768.0000 max=32954 max_rank=0 min=32666 min_rank=3 imbalance_ratio=1.005676 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=8 tokens_per_ep_rank=[32954, 32697, 32700, 32666, 32757, 32911, 32753, 32706] avg=32768.0000 max=32954 max_rank=0 min=32666 min_rank=3 imbalance_ratio=1.005676 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=7 tokens_per_ep_rank=[32971, 33018, 32966, 32453, 32518, 32793, 32634, 32791] avg=32768.0000 max=33018 max_rank=1 min=32453 min_rank=3 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=8 tokens_per_ep_rank=[32954, 32697, 32700, 32666, 32757, 32911, 32753, 32706] avg=32768.0000 max=32954 max_rank=0 min=32666 min_rank=3 imbalance_ratio=1.005676 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=7 tokens_per_ep_rank=[32971, 33018, 32966, 32453, 32518, 32793, 32634, 32791] avg=32768.0000 max=33018 max_rank=1 min=32453 min_rank=3 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=7 tokens_per_ep_rank=[32971, 33018, 32966, 32453, 32518, 32793, 32634, 32791] avg=32768.0000 max=33018 max_rank=1 min=32453 min_rank=3 imbalance_ratio=1.007629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=7 tokens_per_ep_rank=[32971, 33018, 32966, 32453, 32518, 32793, 32634, 32791] avg=32768.0000 max=33018 max_rank=1 min=32453 min_rank=3 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=8 tokens_per_ep_rank=[32954, 32697, 32700, 32666, 32757, 32911, 32753, 32706] avg=32768.0000 max=32954 max_rank=0 min=32666 min_rank=3 imbalance_ratio=1.005676 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=9 tokens_per_ep_rank=[32853, 32918, 32465, 32763, 32776, 32956, 32616, 32797] avg=32768.0000 max=32956 max_rank=5 min=32465 min_rank=2 imbalance_ratio=1.005737 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=8 tokens_per_ep_rank=[32954, 32697, 32700, 32666, 32757, 32911, 32753, 32706] avg=32768.0000 max=32954 max_rank=0 min=32666 min_rank=3 imbalance_ratio=1.005676 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=8 tokens_per_ep_rank=[32954, 32697, 32700, 32666, 32757, 32911, 32753, 32706] avg=32768.0000 max=32954 max_rank=0 min=32666 min_rank=3 imbalance_ratio=1.005676 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=7 tokens_per_ep_rank=[32971, 33018, 32966, 32453, 32518, 32793, 32634, 32791] avg=32768.0000 max=33018 max_rank=1 min=32453 min_rank=3 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=9 tokens_per_ep_rank=[32853, 32918, 32465, 32763, 32776, 32956, 32616, 32797] avg=32768.0000 max=32956 max_rank=5 min=32465 min_rank=2 imbalance_ratio=1.005737 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=9 tokens_per_ep_rank=[32853, 32918, 32465, 32763, 32776, 32956, 32616, 32797] avg=32768.0000 max=32956 max_rank=5 min=32465 min_rank=2 imbalance_ratio=1.005737 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=8 tokens_per_ep_rank=[32954, 32697, 32700, 32666, 32757, 32911, 32753, 32706] avg=32768.0000 max=32954 max_rank=0 min=32666 min_rank=3 imbalance_ratio=1.005676 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=9 tokens_per_ep_rank=[32853, 32918, 32465, 32763, 32776, 32956, 32616, 32797] avg=32768.0000 max=32956 max_rank=5 min=32465 min_rank=2 imbalance_ratio=1.005737 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=8 tokens_per_ep_rank=[32954, 32697, 32700, 32666, 32757, 32911, 32753, 32706] avg=32768.0000 max=32954 max_rank=0 min=32666 min_rank=3 imbalance_ratio=1.005676 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=9 tokens_per_ep_rank=[32853, 32918, 32465, 32763, 32776, 32956, 32616, 32797] avg=32768.0000 max=32956 max_rank=5 min=32465 min_rank=2 imbalance_ratio=1.005737 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=8 tokens_per_ep_rank=[32954, 32697, 32700, 32666, 32757, 32911, 32753, 32706] avg=32768.0000 max=32954 max_rank=0 min=32666 min_rank=3 imbalance_ratio=1.005676 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=9 tokens_per_ep_rank=[32853, 32918, 32465, 32763, 32776, 32956, 32616, 32797] avg=32768.0000 max=32956 max_rank=5 min=32465 min_rank=2 imbalance_ratio=1.005737 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=8 tokens_per_ep_rank=[32954, 32697, 32700, 32666, 32757, 32911, 32753, 32706] avg=32768.0000 max=32954 max_rank=0 min=32666 min_rank=3 imbalance_ratio=1.005676 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=8 tokens_per_ep_rank=[32954, 32697, 32700, 32666, 32757, 32911, 32753, 32706] avg=32768.0000 max=32954 max_rank=0 min=32666 min_rank=3 imbalance_ratio=1.005676 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=10 tokens_per_ep_rank=[32955, 32898, 32306, 32813, 32759, 32777, 32800, 32836] avg=32768.0000 max=32955 max_rank=0 min=32306 min_rank=2 imbalance_ratio=1.005707 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=9 tokens_per_ep_rank=[32853, 32918, 32465, 32763, 32776, 32956, 32616, 32797] avg=32768.0000 max=32956 max_rank=5 min=32465 min_rank=2 imbalance_ratio=1.005737 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=8 tokens_per_ep_rank=[32954, 32697, 32700, 32666, 32757, 32911, 32753, 32706] avg=32768.0000 max=32954 max_rank=0 min=32666 min_rank=3 imbalance_ratio=1.005676 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=9 tokens_per_ep_rank=[32853, 32918, 32465, 32763, 32776, 32956, 32616, 32797] avg=32768.0000 max=32956 max_rank=5 min=32465 min_rank=2 imbalance_ratio=1.005737 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=9 tokens_per_ep_rank=[32853, 32918, 32465, 32763, 32776, 32956, 32616, 32797] avg=32768.0000 max=32956 max_rank=5 min=32465 min_rank=2 imbalance_ratio=1.005737 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=8 tokens_per_ep_rank=[32954, 32697, 32700, 32666, 32757, 32911, 32753, 32706] avg=32768.0000 max=32954 max_rank=0 min=32666 min_rank=3 imbalance_ratio=1.005676 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=10 tokens_per_ep_rank=[32955, 32898, 32306, 32813, 32759, 32777, 32800, 32836] avg=32768.0000 max=32955 max_rank=0 min=32306 min_rank=2 imbalance_ratio=1.005707 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=9 tokens_per_ep_rank=[32853, 32918, 32465, 32763, 32776, 32956, 32616, 32797] avg=32768.0000 max=32956 max_rank=5 min=32465 min_rank=2 imbalance_ratio=1.005737 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=10 tokens_per_ep_rank=[32955, 32898, 32306, 32813, 32759, 32777, 32800, 32836] avg=32768.0000 max=32955 max_rank=0 min=32306 min_rank=2 imbalance_ratio=1.005707 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=10 tokens_per_ep_rank=[32955, 32898, 32306, 32813, 32759, 32777, 32800, 32836] avg=32768.0000 max=32955 max_rank=0 min=32306 min_rank=2 imbalance_ratio=1.005707 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=10 tokens_per_ep_rank=[32955, 32898, 32306, 32813, 32759, 32777, 32800, 32836] avg=32768.0000 max=32955 max_rank=0 min=32306 min_rank=2 imbalance_ratio=1.005707 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=9 tokens_per_ep_rank=[32853, 32918, 32465, 32763, 32776, 32956, 32616, 32797] avg=32768.0000 max=32956 max_rank=5 min=32465 min_rank=2 imbalance_ratio=1.005737 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=9 tokens_per_ep_rank=[32853, 32918, 32465, 32763, 32776, 32956, 32616, 32797] avg=32768.0000 max=32956 max_rank=5 min=32465 min_rank=2 imbalance_ratio=1.005737 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=9 tokens_per_ep_rank=[32853, 32918, 32465, 32763, 32776, 32956, 32616, 32797] avg=32768.0000 max=32956 max_rank=5 min=32465 min_rank=2 imbalance_ratio=1.005737 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=10 tokens_per_ep_rank=[32955, 32898, 32306, 32813, 32759, 32777, 32800, 32836] avg=32768.0000 max=32955 max_rank=0 min=32306 min_rank=2 imbalance_ratio=1.005707 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=10 tokens_per_ep_rank=[32955, 32898, 32306, 32813, 32759, 32777, 32800, 32836] avg=32768.0000 max=32955 max_rank=0 min=32306 min_rank=2 imbalance_ratio=1.005707 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=9 tokens_per_ep_rank=[32853, 32918, 32465, 32763, 32776, 32956, 32616, 32797] avg=32768.0000 max=32956 max_rank=5 min=32465 min_rank=2 imbalance_ratio=1.005737 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=10 tokens_per_ep_rank=[32955, 32898, 32306, 32813, 32759, 32777, 32800, 32836] avg=32768.0000 max=32955 max_rank=0 min=32306 min_rank=2 imbalance_ratio=1.005707 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=9 tokens_per_ep_rank=[32853, 32918, 32465, 32763, 32776, 32956, 32616, 32797] avg=32768.0000 max=32956 max_rank=5 min=32465 min_rank=2 imbalance_ratio=1.005737 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=10 tokens_per_ep_rank=[32955, 32898, 32306, 32813, 32759, 32777, 32800, 32836] avg=32768.0000 max=32955 max_rank=0 min=32306 min_rank=2 imbalance_ratio=1.005707 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=9 tokens_per_ep_rank=[32853, 32918, 32465, 32763, 32776, 32956, 32616, 32797] avg=32768.0000 max=32956 max_rank=5 min=32465 min_rank=2 imbalance_ratio=1.005737 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=10 tokens_per_ep_rank=[32955, 32898, 32306, 32813, 32759, 32777, 32800, 32836] avg=32768.0000 max=32955 max_rank=0 min=32306 min_rank=2 imbalance_ratio=1.005707 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=10 tokens_per_ep_rank=[32955, 32898, 32306, 32813, 32759, 32777, 32800, 32836] avg=32768.0000 max=32955 max_rank=0 min=32306 min_rank=2 imbalance_ratio=1.005707 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=10 tokens_per_ep_rank=[32955, 32898, 32306, 32813, 32759, 32777, 32800, 32836] avg=32768.0000 max=32955 max_rank=0 min=32306 min_rank=2 imbalance_ratio=1.005707 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=10 tokens_per_ep_rank=[32955, 32898, 32306, 32813, 32759, 32777, 32800, 32836] avg=32768.0000 max=32955 max_rank=0 min=32306 min_rank=2 imbalance_ratio=1.005707 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=10 tokens_per_ep_rank=[32955, 32898, 32306, 32813, 32759, 32777, 32800, 32836] avg=32768.0000 max=32955 max_rank=0 min=32306 min_rank=2 imbalance_ratio=1.005707 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=10 tokens_per_ep_rank=[32955, 32898, 32306, 32813, 32759, 32777, 32800, 32836] avg=32768.0000 max=32955 max_rank=0 min=32306 min_rank=2 imbalance_ratio=1.005707 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=10 tokens_per_ep_rank=[32955, 32898, 32306, 32813, 32759, 32777, 32800, 32836] avg=32768.0000 max=32955 max_rank=0 min=32306 min_rank=2 imbalance_ratio=1.005707 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=4 records=10 total_expected_recv=327736 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76032 peak_demand=32956 required_capacity=1.005737 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=4 records=10 total_expected_recv=327782 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76416 peak_demand=32972 required_capacity=1.006226 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=4 records=10 total_expected_recv=327057 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=77440 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=4 records=10 total_expected_recv=327763 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76928 peak_demand=32986 required_capacity=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=4 records=10 total_expected_recv=326955 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=77824 peak_demand=32800 required_capacity=1.000977 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=4 records=10 total_expected_recv=327354 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=77184 peak_demand=32997 required_capacity=1.006989 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=4 records=10 total_expected_recv=328732 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=75264 peak_demand=33060 required_capacity=1.008911 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=4 records=10 total_expected_recv=328061 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76288 peak_demand=32971 required_capacity=1.006195 + [2026-08-26 15:53:12] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 213.1 | throughput per GPU (TFLOP/s/GPU): 579.4 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 9.860033E+00 | loss scale: 1.0 | grad norm: 18.911 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 33664 | min safe capacity: 1.0273 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=0 calls=1 budget=40960 expected_recv=32591 expected_overflow=0 exact_recv_per_rank=[32716, 32818, 32591, 33096, 32750, 32706, 32685, 32782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32716, 32818, 32591, 33096, 32750, 32706, 32685, 32782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=0 calls=1 budget=40960 expected_recv=32706 expected_overflow=0 exact_recv_per_rank=[32716, 32818, 32591, 33096, 32750, 32706, 32685, 32782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=0 calls=1 budget=40960 expected_recv=32818 expected_overflow=0 exact_recv_per_rank=[32716, 32818, 32591, 33096, 32750, 32706, 32685, 32782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=0 calls=1 budget=40960 expected_recv=32685 expected_overflow=0 exact_recv_per_rank=[32716, 32818, 32591, 33096, 32750, 32706, 32685, 32782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=0 calls=1 budget=40960 expected_recv=32750 expected_overflow=0 exact_recv_per_rank=[32716, 32818, 32591, 33096, 32750, 32706, 32685, 32782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=0 calls=1 budget=40960 expected_recv=32782 expected_overflow=0 exact_recv_per_rank=[32716, 32818, 32591, 33096, 32750, 32706, 32685, 32782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=0 calls=1 budget=40960 expected_recv=32591 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=0 calls=1 budget=40960 expected_recv=32706 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=0 calls=1 budget=40960 expected_recv=32716 expected_overflow=0 exact_recv_per_rank=[32716, 32818, 32591, 33096, 32750, 32706, 32685, 32782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=0 calls=1 budget=40960 expected_recv=32818 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=0 calls=1 budget=40960 expected_recv=32685 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=0 calls=1 budget=40960 expected_recv=32750 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=0 calls=1 budget=40960 expected_recv=32782 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=0 calls=1 budget=40960 expected_recv=32716 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=1 calls=1 budget=40960 expected_recv=32930 expected_overflow=0 exact_recv_per_rank=[32875, 32582, 32930, 32981, 32747, 33035, 32453, 32541] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=1 calls=1 budget=40960 expected_recv=33035 expected_overflow=0 exact_recv_per_rank=[32875, 32582, 32930, 32981, 32747, 33035, 32453, 32541] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=1 calls=1 budget=40960 expected_recv=32981 expected_overflow=0 exact_recv_per_rank=[32875, 32582, 32930, 32981, 32747, 33035, 32453, 32541] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=1 calls=1 budget=40960 expected_recv=32582 expected_overflow=0 exact_recv_per_rank=[32875, 32582, 32930, 32981, 32747, 33035, 32453, 32541] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=1 calls=1 budget=40960 expected_recv=32453 expected_overflow=0 exact_recv_per_rank=[32875, 32582, 32930, 32981, 32747, 33035, 32453, 32541] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=1 calls=1 budget=40960 expected_recv=32930 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=1 calls=1 budget=40960 expected_recv=32541 expected_overflow=0 exact_recv_per_rank=[32875, 32582, 32930, 32981, 32747, 33035, 32453, 32541] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=1 calls=1 budget=40960 expected_recv=32747 expected_overflow=0 exact_recv_per_rank=[32875, 32582, 32930, 32981, 32747, 33035, 32453, 32541] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=1 calls=1 budget=40960 expected_recv=33035 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=1 calls=1 budget=40960 expected_recv=32981 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=1 calls=1 budget=40960 expected_recv=32582 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=1 calls=1 budget=40960 expected_recv=32453 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=1 calls=1 budget=40960 expected_recv=32875 expected_overflow=0 exact_recv_per_rank=[32875, 32582, 32930, 32981, 32747, 33035, 32453, 32541] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=1 calls=1 budget=40960 expected_recv=32541 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=1 calls=1 budget=40960 expected_recv=32747 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=1 calls=1 budget=40960 expected_recv=32875 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=2 calls=1 budget=40960 expected_recv=32678 expected_overflow=0 exact_recv_per_rank=[32881, 32743, 32678, 32963, 32723, 32817, 32863, 32476] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=2 calls=1 budget=40960 expected_recv=32963 expected_overflow=0 exact_recv_per_rank=[32881, 32743, 32678, 32963, 32723, 32817, 32863, 32476] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=2 calls=1 budget=40960 expected_recv=32817 expected_overflow=0 exact_recv_per_rank=[32881, 32743, 32678, 32963, 32723, 32817, 32863, 32476] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=2 calls=1 budget=40960 expected_recv=32743 expected_overflow=0 exact_recv_per_rank=[32881, 32743, 32678, 32963, 32723, 32817, 32863, 32476] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=2 calls=1 budget=40960 expected_recv=32863 expected_overflow=0 exact_recv_per_rank=[32881, 32743, 32678, 32963, 32723, 32817, 32863, 32476] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=2 calls=1 budget=40960 expected_recv=32476 expected_overflow=0 exact_recv_per_rank=[32881, 32743, 32678, 32963, 32723, 32817, 32863, 32476] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=2 calls=1 budget=40960 expected_recv=32723 expected_overflow=0 exact_recv_per_rank=[32881, 32743, 32678, 32963, 32723, 32817, 32863, 32476] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=2 calls=1 budget=40960 expected_recv=32678 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=2 calls=1 budget=40960 expected_recv=32963 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=2 calls=1 budget=40960 expected_recv=32817 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=2 calls=1 budget=40960 expected_recv=32743 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=2 calls=1 budget=40960 expected_recv=32863 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=2 calls=1 budget=40960 expected_recv=32881 expected_overflow=0 exact_recv_per_rank=[32881, 32743, 32678, 32963, 32723, 32817, 32863, 32476] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=2 calls=1 budget=40960 expected_recv=32476 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=2 calls=1 budget=40960 expected_recv=32723 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=2 calls=1 budget=40960 expected_recv=32881 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=3 calls=1 budget=40960 expected_recv=33015 expected_overflow=0 exact_recv_per_rank=[33078, 32676, 32607, 33015, 32865, 32722, 32632, 32549] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=3 calls=1 budget=40960 expected_recv=32607 expected_overflow=0 exact_recv_per_rank=[33078, 32676, 32607, 33015, 32865, 32722, 32632, 32549] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=3 calls=1 budget=40960 expected_recv=32722 expected_overflow=0 exact_recv_per_rank=[33078, 32676, 32607, 33015, 32865, 32722, 32632, 32549] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=3 calls=1 budget=40960 expected_recv=32676 expected_overflow=0 exact_recv_per_rank=[33078, 32676, 32607, 33015, 32865, 32722, 32632, 32549] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=3 calls=1 budget=40960 expected_recv=33015 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=3 calls=1 budget=40960 expected_recv=32632 expected_overflow=0 exact_recv_per_rank=[33078, 32676, 32607, 33015, 32865, 32722, 32632, 32549] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=3 calls=1 budget=40960 expected_recv=32549 expected_overflow=0 exact_recv_per_rank=[33078, 32676, 32607, 33015, 32865, 32722, 32632, 32549] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=3 calls=1 budget=40960 expected_recv=32865 expected_overflow=0 exact_recv_per_rank=[33078, 32676, 32607, 33015, 32865, 32722, 32632, 32549] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=3 calls=1 budget=40960 expected_recv=32607 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=3 calls=1 budget=40960 expected_recv=32722 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=3 calls=1 budget=40960 expected_recv=32676 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=3 calls=1 budget=40960 expected_recv=33078 expected_overflow=0 exact_recv_per_rank=[33078, 32676, 32607, 33015, 32865, 32722, 32632, 32549] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=3 calls=1 budget=40960 expected_recv=32632 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=3 calls=1 budget=40960 expected_recv=32549 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=3 calls=1 budget=40960 expected_recv=32865 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=3 calls=1 budget=40960 expected_recv=33078 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=4 calls=1 budget=40960 expected_recv=32886 expected_overflow=0 exact_recv_per_rank=[32741, 32770, 32669, 32886, 32946, 32763, 32547, 32822] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=4 calls=1 budget=40960 expected_recv=32669 expected_overflow=0 exact_recv_per_rank=[32741, 32770, 32669, 32886, 32946, 32763, 32547, 32822] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=4 calls=1 budget=40960 expected_recv=32886 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=4 calls=1 budget=40960 expected_recv=32763 expected_overflow=0 exact_recv_per_rank=[32741, 32770, 32669, 32886, 32946, 32763, 32547, 32822] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=4 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[32741, 32770, 32669, 32886, 32946, 32763, 32547, 32822] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=4 calls=1 budget=40960 expected_recv=32822 expected_overflow=0 exact_recv_per_rank=[32741, 32770, 32669, 32886, 32946, 32763, 32547, 32822] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=4 calls=1 budget=40960 expected_recv=32547 expected_overflow=0 exact_recv_per_rank=[32741, 32770, 32669, 32886, 32946, 32763, 32547, 32822] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=4 calls=1 budget=40960 expected_recv=32946 expected_overflow=0 exact_recv_per_rank=[32741, 32770, 32669, 32886, 32946, 32763, 32547, 32822] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=4 calls=1 budget=40960 expected_recv=32669 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=4 calls=1 budget=40960 expected_recv=32763 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=4 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=4 calls=1 budget=40960 expected_recv=32741 expected_overflow=0 exact_recv_per_rank=[32741, 32770, 32669, 32886, 32946, 32763, 32547, 32822] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=4 calls=1 budget=40960 expected_recv=32822 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=4 calls=1 budget=40960 expected_recv=32547 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=4 calls=1 budget=40960 expected_recv=32946 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=4 calls=1 budget=40960 expected_recv=32741 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=5 calls=1 budget=40960 expected_recv=32796 expected_overflow=0 exact_recv_per_rank=[32760, 32816, 33057, 32796, 32437, 32710, 32905, 32663] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=5 calls=1 budget=40960 expected_recv=32796 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=5 calls=1 budget=40960 expected_recv=33057 expected_overflow=0 exact_recv_per_rank=[32760, 32816, 33057, 32796, 32437, 32710, 32905, 32663] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=5 calls=1 budget=40960 expected_recv=32710 expected_overflow=0 exact_recv_per_rank=[32760, 32816, 33057, 32796, 32437, 32710, 32905, 32663] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=5 calls=1 budget=40960 expected_recv=32816 expected_overflow=0 exact_recv_per_rank=[32760, 32816, 33057, 32796, 32437, 32710, 32905, 32663] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=5 calls=1 budget=40960 expected_recv=32663 expected_overflow=0 exact_recv_per_rank=[32760, 32816, 33057, 32796, 32437, 32710, 32905, 32663] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=5 calls=1 budget=40960 expected_recv=32437 expected_overflow=0 exact_recv_per_rank=[32760, 32816, 33057, 32796, 32437, 32710, 32905, 32663] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=5 calls=1 budget=40960 expected_recv=32905 expected_overflow=0 exact_recv_per_rank=[32760, 32816, 33057, 32796, 32437, 32710, 32905, 32663] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=5 calls=1 budget=40960 expected_recv=33057 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=5 calls=1 budget=40960 expected_recv=32760 expected_overflow=0 exact_recv_per_rank=[32760, 32816, 33057, 32796, 32437, 32710, 32905, 32663] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=5 calls=1 budget=40960 expected_recv=32710 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=5 calls=1 budget=40960 expected_recv=32816 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=5 calls=1 budget=40960 expected_recv=32663 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=5 calls=1 budget=40960 expected_recv=32437 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=5 calls=1 budget=40960 expected_recv=32905 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=6 calls=1 budget=40960 expected_recv=32895 expected_overflow=0 exact_recv_per_rank=[32678, 32911, 32767, 32895, 32892, 32745, 32580, 32676] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=5 calls=1 budget=40960 expected_recv=32760 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=6 calls=1 budget=40960 expected_recv=32895 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=6 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32678, 32911, 32767, 32895, 32892, 32745, 32580, 32676] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=6 calls=1 budget=40960 expected_recv=32911 expected_overflow=0 exact_recv_per_rank=[32678, 32911, 32767, 32895, 32892, 32745, 32580, 32676] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=6 calls=1 budget=40960 expected_recv=32676 expected_overflow=0 exact_recv_per_rank=[32678, 32911, 32767, 32895, 32892, 32745, 32580, 32676] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=6 calls=1 budget=40960 expected_recv=32745 expected_overflow=0 exact_recv_per_rank=[32678, 32911, 32767, 32895, 32892, 32745, 32580, 32676] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=6 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=6 calls=1 budget=40960 expected_recv=32892 expected_overflow=0 exact_recv_per_rank=[32678, 32911, 32767, 32895, 32892, 32745, 32580, 32676] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=6 calls=1 budget=40960 expected_recv=32580 expected_overflow=0 exact_recv_per_rank=[32678, 32911, 32767, 32895, 32892, 32745, 32580, 32676] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=6 calls=1 budget=40960 expected_recv=32911 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=6 calls=1 budget=40960 expected_recv=32676 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=6 calls=1 budget=40960 expected_recv=32745 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=6 calls=1 budget=40960 expected_recv=32678 expected_overflow=0 exact_recv_per_rank=[32678, 32911, 32767, 32895, 32892, 32745, 32580, 32676] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=6 calls=1 budget=40960 expected_recv=32892 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=6 calls=1 budget=40960 expected_recv=32580 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=7 calls=1 budget=40960 expected_recv=32730 expected_overflow=0 exact_recv_per_rank=[32701, 33096, 32732, 32730, 32764, 32622, 32820, 32679] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=6 calls=1 budget=40960 expected_recv=32678 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=7 calls=1 budget=40960 expected_recv=32730 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=7 calls=1 budget=40960 expected_recv=32732 expected_overflow=0 exact_recv_per_rank=[32701, 33096, 32732, 32730, 32764, 32622, 32820, 32679] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32701, 33096, 32732, 32730, 32764, 32622, 32820, 32679] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=7 calls=1 budget=40960 expected_recv=32679 expected_overflow=0 exact_recv_per_rank=[32701, 33096, 32732, 32730, 32764, 32622, 32820, 32679] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=7 calls=1 budget=40960 expected_recv=32622 expected_overflow=0 exact_recv_per_rank=[32701, 33096, 32732, 32730, 32764, 32622, 32820, 32679] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=7 calls=1 budget=40960 expected_recv=32732 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=7 calls=1 budget=40960 expected_recv=32764 expected_overflow=0 exact_recv_per_rank=[32701, 33096, 32732, 32730, 32764, 32622, 32820, 32679] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=7 calls=1 budget=40960 expected_recv=32679 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=7 calls=1 budget=40960 expected_recv=32820 expected_overflow=0 exact_recv_per_rank=[32701, 33096, 32732, 32730, 32764, 32622, 32820, 32679] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=7 calls=1 budget=40960 expected_recv=32622 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=7 calls=1 budget=40960 expected_recv=32764 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=7 calls=1 budget=40960 expected_recv=32701 expected_overflow=0 exact_recv_per_rank=[32701, 33096, 32732, 32730, 32764, 32622, 32820, 32679] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=7 calls=1 budget=40960 expected_recv=32820 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=8 calls=1 budget=40960 expected_recv=33033 expected_overflow=0 exact_recv_per_rank=[32807, 32890, 32824, 33033, 32537, 32606, 32647, 32800] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=7 calls=1 budget=40960 expected_recv=32701 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=8 calls=1 budget=40960 expected_recv=33033 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=8 calls=1 budget=40960 expected_recv=32824 expected_overflow=0 exact_recv_per_rank=[32807, 32890, 32824, 33033, 32537, 32606, 32647, 32800] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=8 calls=1 budget=40960 expected_recv=32800 expected_overflow=0 exact_recv_per_rank=[32807, 32890, 32824, 33033, 32537, 32606, 32647, 32800] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=8 calls=1 budget=40960 expected_recv=32824 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=8 calls=1 budget=40960 expected_recv=32890 expected_overflow=0 exact_recv_per_rank=[32807, 32890, 32824, 33033, 32537, 32606, 32647, 32800] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=8 calls=1 budget=40960 expected_recv=32606 expected_overflow=0 exact_recv_per_rank=[32807, 32890, 32824, 33033, 32537, 32606, 32647, 32800] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=8 calls=1 budget=40960 expected_recv=32800 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=8 calls=1 budget=40960 expected_recv=32537 expected_overflow=0 exact_recv_per_rank=[32807, 32890, 32824, 33033, 32537, 32606, 32647, 32800] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=8 calls=1 budget=40960 expected_recv=32890 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=8 calls=1 budget=40960 expected_recv=32606 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=8 calls=1 budget=40960 expected_recv=32647 expected_overflow=0 exact_recv_per_rank=[32807, 32890, 32824, 33033, 32537, 32606, 32647, 32800] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=8 calls=1 budget=40960 expected_recv=32537 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=8 calls=1 budget=40960 expected_recv=32807 expected_overflow=0 exact_recv_per_rank=[32807, 32890, 32824, 33033, 32537, 32606, 32647, 32800] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=9 calls=1 budget=40960 expected_recv=32622 expected_overflow=0 exact_recv_per_rank=[32792, 32723, 32818, 32622, 32875, 32684, 32774, 32856] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=8 calls=1 budget=40960 expected_recv=32647 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=8 calls=1 budget=40960 expected_recv=32807 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=9 calls=1 budget=40960 expected_recv=32622 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=9 calls=1 budget=40960 expected_recv=32818 expected_overflow=0 exact_recv_per_rank=[32792, 32723, 32818, 32622, 32875, 32684, 32774, 32856] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=9 calls=1 budget=40960 expected_recv=32856 expected_overflow=0 exact_recv_per_rank=[32792, 32723, 32818, 32622, 32875, 32684, 32774, 32856] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=9 calls=1 budget=40960 expected_recv=32818 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=9 calls=1 budget=40960 expected_recv=32723 expected_overflow=0 exact_recv_per_rank=[32792, 32723, 32818, 32622, 32875, 32684, 32774, 32856] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=9 calls=1 budget=40960 expected_recv=32684 expected_overflow=0 exact_recv_per_rank=[32792, 32723, 32818, 32622, 32875, 32684, 32774, 32856] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=9 calls=1 budget=40960 expected_recv=32875 expected_overflow=0 exact_recv_per_rank=[32792, 32723, 32818, 32622, 32875, 32684, 32774, 32856] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=9 calls=1 budget=40960 expected_recv=32856 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=9 calls=1 budget=40960 expected_recv=32723 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=9 calls=1 budget=40960 expected_recv=32684 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=9 calls=1 budget=40960 expected_recv=32875 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=9 calls=1 budget=40960 expected_recv=32774 expected_overflow=0 exact_recv_per_rank=[32792, 32723, 32818, 32622, 32875, 32684, 32774, 32856] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=9 calls=1 budget=40960 expected_recv=32792 expected_overflow=0 exact_recv_per_rank=[32792, 32723, 32818, 32622, 32875, 32684, 32774, 32856] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=9 calls=1 budget=40960 expected_recv=32774 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=1 tokens_per_ep_rank=[32716, 32818, 32497, 33190, 32750, 32706, 32685, 32782] avg=32768.0000 max=33190 max_rank=3 min=32497 min_rank=2 imbalance_ratio=1.012878 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=9 calls=1 budget=40960 expected_recv=32792 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=1 tokens_per_ep_rank=[32716, 32818, 32497, 33190, 32750, 32706, 32685, 32782] avg=32768.0000 max=33190 max_rank=3 min=32497 min_rank=2 imbalance_ratio=1.012878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=1 tokens_per_ep_rank=[32716, 32818, 32591, 33096, 32750, 32706, 32685, 32782] avg=32768.0000 max=33096 max_rank=3 min=32591 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=94 max_load_reduction_pct=0.002832 moved_to_replicas=94 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=1 tokens_per_ep_rank=[32716, 32818, 32497, 33190, 32750, 32706, 32685, 32782] avg=32768.0000 max=33190 max_rank=3 min=32497 min_rank=2 imbalance_ratio=1.012878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=1 tokens_per_ep_rank=[32716, 32818, 32497, 33190, 32750, 32706, 32685, 32782] avg=32768.0000 max=33190 max_rank=3 min=32497 min_rank=2 imbalance_ratio=1.012878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=1 tokens_per_ep_rank=[32716, 32818, 32497, 33190, 32750, 32706, 32685, 32782] avg=32768.0000 max=33190 max_rank=3 min=32497 min_rank=2 imbalance_ratio=1.012878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=1 tokens_per_ep_rank=[32716, 32818, 32497, 33190, 32750, 32706, 32685, 32782] avg=32768.0000 max=33190 max_rank=3 min=32497 min_rank=2 imbalance_ratio=1.012878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=1 tokens_per_ep_rank=[32716, 32818, 32497, 33190, 32750, 32706, 32685, 32782] avg=32768.0000 max=33190 max_rank=3 min=32497 min_rank=2 imbalance_ratio=1.012878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=1 tokens_per_ep_rank=[32716, 32818, 32591, 33096, 32750, 32706, 32685, 32782] avg=32768.0000 max=33096 max_rank=3 min=32591 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=94 max_load_reduction_pct=0.002832 moved_to_replicas=94 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=1 tokens_per_ep_rank=[32716, 32818, 32591, 33096, 32750, 32706, 32685, 32782] avg=32768.0000 max=33096 max_rank=3 min=32591 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=94 max_load_reduction_pct=0.002832 moved_to_replicas=94 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=1 tokens_per_ep_rank=[32716, 32818, 32591, 33096, 32750, 32706, 32685, 32782] avg=32768.0000 max=33096 max_rank=3 min=32591 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=94 max_load_reduction_pct=0.002832 moved_to_replicas=94 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=1 tokens_per_ep_rank=[32716, 32818, 32591, 33096, 32750, 32706, 32685, 32782] avg=32768.0000 max=33096 max_rank=3 min=32591 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=94 max_load_reduction_pct=0.002832 moved_to_replicas=94 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=1 tokens_per_ep_rank=[32716, 32818, 32497, 33190, 32750, 32706, 32685, 32782] avg=32768.0000 max=33190 max_rank=3 min=32497 min_rank=2 imbalance_ratio=1.012878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=1 tokens_per_ep_rank=[32716, 32818, 32591, 33096, 32750, 32706, 32685, 32782] avg=32768.0000 max=33096 max_rank=3 min=32591 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=94 max_load_reduction_pct=0.002832 moved_to_replicas=94 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=2 tokens_per_ep_rank=[32875, 32582, 32930, 32981, 32747, 33035, 32453, 32541] avg=32768.0000 max=33035 max_rank=5 min=32453 min_rank=6 imbalance_ratio=1.008148 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=1 tokens_per_ep_rank=[32716, 32818, 32591, 33096, 32750, 32706, 32685, 32782] avg=32768.0000 max=33096 max_rank=3 min=32591 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=94 max_load_reduction_pct=0.002832 moved_to_replicas=94 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=2 tokens_per_ep_rank=[32875, 32582, 32930, 32981, 32747, 33035, 32453, 32541] avg=32768.0000 max=33035 max_rank=5 min=32453 min_rank=6 imbalance_ratio=1.008148 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=1 tokens_per_ep_rank=[32716, 32818, 32591, 33096, 32750, 32706, 32685, 32782] avg=32768.0000 max=33096 max_rank=3 min=32591 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=94 max_load_reduction_pct=0.002832 moved_to_replicas=94 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=2 tokens_per_ep_rank=[32875, 32582, 32930, 32981, 32747, 33035, 32453, 32541] avg=32768.0000 max=33035 max_rank=5 min=32453 min_rank=6 imbalance_ratio=1.008148 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=2 tokens_per_ep_rank=[32875, 32582, 32930, 32981, 32747, 33035, 32453, 32541] avg=32768.0000 max=33035 max_rank=5 min=32453 min_rank=6 imbalance_ratio=1.008148 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=2 tokens_per_ep_rank=[32875, 32582, 32930, 32981, 32747, 33035, 32453, 32541] avg=32768.0000 max=33035 max_rank=5 min=32453 min_rank=6 imbalance_ratio=1.008148 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=2 tokens_per_ep_rank=[32875, 32582, 32930, 32981, 32747, 33035, 32453, 32541] avg=32768.0000 max=33035 max_rank=5 min=32453 min_rank=6 imbalance_ratio=1.008148 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=2 tokens_per_ep_rank=[32875, 32582, 32930, 32981, 32747, 33035, 32453, 32541] avg=32768.0000 max=33035 max_rank=5 min=32453 min_rank=6 imbalance_ratio=1.008148 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=2 tokens_per_ep_rank=[32875, 32582, 32930, 32981, 32747, 33035, 32453, 32541] avg=32768.0000 max=33035 max_rank=5 min=32453 min_rank=6 imbalance_ratio=1.008148 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=2 tokens_per_ep_rank=[32875, 32582, 32930, 32981, 32747, 33035, 32453, 32541] avg=32768.0000 max=33035 max_rank=5 min=32453 min_rank=6 imbalance_ratio=1.008148 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=2 tokens_per_ep_rank=[32875, 32582, 32930, 32981, 32747, 33035, 32453, 32541] avg=32768.0000 max=33035 max_rank=5 min=32453 min_rank=6 imbalance_ratio=1.008148 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=2 tokens_per_ep_rank=[32875, 32582, 32930, 32981, 32747, 33035, 32453, 32541] avg=32768.0000 max=33035 max_rank=5 min=32453 min_rank=6 imbalance_ratio=1.008148 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=2 tokens_per_ep_rank=[32875, 32582, 32930, 32981, 32747, 33035, 32453, 32541] avg=32768.0000 max=33035 max_rank=5 min=32453 min_rank=6 imbalance_ratio=1.008148 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=2 tokens_per_ep_rank=[32875, 32582, 32930, 32981, 32747, 33035, 32453, 32541] avg=32768.0000 max=33035 max_rank=5 min=32453 min_rank=6 imbalance_ratio=1.008148 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=2 tokens_per_ep_rank=[32875, 32582, 32930, 32981, 32747, 33035, 32453, 32541] avg=32768.0000 max=33035 max_rank=5 min=32453 min_rank=6 imbalance_ratio=1.008148 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=3 tokens_per_ep_rank=[32881, 32743, 32678, 32963, 32723, 32817, 32863, 32476] avg=32768.0000 max=32963 max_rank=3 min=32476 min_rank=7 imbalance_ratio=1.005951 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=2 tokens_per_ep_rank=[32875, 32582, 32930, 32981, 32747, 33035, 32453, 32541] avg=32768.0000 max=33035 max_rank=5 min=32453 min_rank=6 imbalance_ratio=1.008148 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=3 tokens_per_ep_rank=[32881, 32743, 32678, 32963, 32723, 32817, 32863, 32476] avg=32768.0000 max=32963 max_rank=3 min=32476 min_rank=7 imbalance_ratio=1.005951 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=3 tokens_per_ep_rank=[32881, 32743, 32678, 32963, 32723, 32817, 32863, 32476] avg=32768.0000 max=32963 max_rank=3 min=32476 min_rank=7 imbalance_ratio=1.005951 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=3 tokens_per_ep_rank=[32881, 32743, 32678, 32963, 32723, 32817, 32863, 32476] avg=32768.0000 max=32963 max_rank=3 min=32476 min_rank=7 imbalance_ratio=1.005951 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=2 tokens_per_ep_rank=[32875, 32582, 32930, 32981, 32747, 33035, 32453, 32541] avg=32768.0000 max=33035 max_rank=5 min=32453 min_rank=6 imbalance_ratio=1.008148 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=3 tokens_per_ep_rank=[32881, 32743, 32678, 32963, 32723, 32817, 32863, 32476] avg=32768.0000 max=32963 max_rank=3 min=32476 min_rank=7 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=3 tokens_per_ep_rank=[32881, 32743, 32678, 32963, 32723, 32817, 32863, 32476] avg=32768.0000 max=32963 max_rank=3 min=32476 min_rank=7 imbalance_ratio=1.005951 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=3 tokens_per_ep_rank=[32881, 32743, 32678, 32963, 32723, 32817, 32863, 32476] avg=32768.0000 max=32963 max_rank=3 min=32476 min_rank=7 imbalance_ratio=1.005951 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=3 tokens_per_ep_rank=[32881, 32743, 32678, 32963, 32723, 32817, 32863, 32476] avg=32768.0000 max=32963 max_rank=3 min=32476 min_rank=7 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=3 tokens_per_ep_rank=[32881, 32743, 32678, 32963, 32723, 32817, 32863, 32476] avg=32768.0000 max=32963 max_rank=3 min=32476 min_rank=7 imbalance_ratio=1.005951 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=3 tokens_per_ep_rank=[32881, 32743, 32678, 32963, 32723, 32817, 32863, 32476] avg=32768.0000 max=32963 max_rank=3 min=32476 min_rank=7 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=3 tokens_per_ep_rank=[32881, 32743, 32678, 32963, 32723, 32817, 32863, 32476] avg=32768.0000 max=32963 max_rank=3 min=32476 min_rank=7 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=3 tokens_per_ep_rank=[32881, 32743, 32678, 32963, 32723, 32817, 32863, 32476] avg=32768.0000 max=32963 max_rank=3 min=32476 min_rank=7 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=3 tokens_per_ep_rank=[32881, 32743, 32678, 32963, 32723, 32817, 32863, 32476] avg=32768.0000 max=32963 max_rank=3 min=32476 min_rank=7 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=3 tokens_per_ep_rank=[32881, 32743, 32678, 32963, 32723, 32817, 32863, 32476] avg=32768.0000 max=32963 max_rank=3 min=32476 min_rank=7 imbalance_ratio=1.005951 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=4 tokens_per_ep_rank=[33078, 32676, 32607, 33015, 32865, 32722, 32632, 32549] avg=32768.0000 max=33078 max_rank=0 min=32549 min_rank=7 imbalance_ratio=1.009460 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=3 tokens_per_ep_rank=[32881, 32743, 32678, 32963, 32723, 32817, 32863, 32476] avg=32768.0000 max=32963 max_rank=3 min=32476 min_rank=7 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=4 tokens_per_ep_rank=[33078, 32676, 32607, 33015, 32865, 32722, 32632, 32549] avg=32768.0000 max=33078 max_rank=0 min=32549 min_rank=7 imbalance_ratio=1.009460 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=4 tokens_per_ep_rank=[33078, 32676, 32607, 33015, 32865, 32722, 32632, 32549] avg=32768.0000 max=33078 max_rank=0 min=32549 min_rank=7 imbalance_ratio=1.009460 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=4 tokens_per_ep_rank=[33078, 32676, 32607, 33015, 32865, 32722, 32632, 32549] avg=32768.0000 max=33078 max_rank=0 min=32549 min_rank=7 imbalance_ratio=1.009460 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=3 tokens_per_ep_rank=[32881, 32743, 32678, 32963, 32723, 32817, 32863, 32476] avg=32768.0000 max=32963 max_rank=3 min=32476 min_rank=7 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=4 tokens_per_ep_rank=[33078, 32676, 32607, 33015, 32865, 32722, 32632, 32549] avg=32768.0000 max=33078 max_rank=0 min=32549 min_rank=7 imbalance_ratio=1.009460 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=4 tokens_per_ep_rank=[33078, 32676, 32607, 33015, 32865, 32722, 32632, 32549] avg=32768.0000 max=33078 max_rank=0 min=32549 min_rank=7 imbalance_ratio=1.009460 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=4 tokens_per_ep_rank=[33078, 32676, 32607, 33015, 32865, 32722, 32632, 32549] avg=32768.0000 max=33078 max_rank=0 min=32549 min_rank=7 imbalance_ratio=1.009460 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=4 tokens_per_ep_rank=[33078, 32676, 32607, 33015, 32865, 32722, 32632, 32549] avg=32768.0000 max=33078 max_rank=0 min=32549 min_rank=7 imbalance_ratio=1.009460 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=4 tokens_per_ep_rank=[33078, 32676, 32607, 33015, 32865, 32722, 32632, 32549] avg=32768.0000 max=33078 max_rank=0 min=32549 min_rank=7 imbalance_ratio=1.009460 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=4 tokens_per_ep_rank=[33078, 32676, 32607, 33015, 32865, 32722, 32632, 32549] avg=32768.0000 max=33078 max_rank=0 min=32549 min_rank=7 imbalance_ratio=1.009460 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=4 tokens_per_ep_rank=[33078, 32676, 32607, 33015, 32865, 32722, 32632, 32549] avg=32768.0000 max=33078 max_rank=0 min=32549 min_rank=7 imbalance_ratio=1.009460 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=4 tokens_per_ep_rank=[33078, 32676, 32607, 33015, 32865, 32722, 32632, 32549] avg=32768.0000 max=33078 max_rank=0 min=32549 min_rank=7 imbalance_ratio=1.009460 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=4 tokens_per_ep_rank=[33078, 32676, 32607, 33015, 32865, 32722, 32632, 32549] avg=32768.0000 max=33078 max_rank=0 min=32549 min_rank=7 imbalance_ratio=1.009460 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=4 tokens_per_ep_rank=[33078, 32676, 32607, 33015, 32865, 32722, 32632, 32549] avg=32768.0000 max=33078 max_rank=0 min=32549 min_rank=7 imbalance_ratio=1.009460 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=5 tokens_per_ep_rank=[32741, 32770, 32669, 32886, 32946, 32763, 32547, 32822] avg=32768.0000 max=32946 max_rank=4 min=32547 min_rank=6 imbalance_ratio=1.005432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=4 tokens_per_ep_rank=[33078, 32676, 32607, 33015, 32865, 32722, 32632, 32549] avg=32768.0000 max=33078 max_rank=0 min=32549 min_rank=7 imbalance_ratio=1.009460 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=5 tokens_per_ep_rank=[32741, 32770, 32669, 32886, 32946, 32763, 32547, 32822] avg=32768.0000 max=32946 max_rank=4 min=32547 min_rank=6 imbalance_ratio=1.005432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=5 tokens_per_ep_rank=[32741, 32770, 32669, 32886, 32946, 32763, 32547, 32822] avg=32768.0000 max=32946 max_rank=4 min=32547 min_rank=6 imbalance_ratio=1.005432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=5 tokens_per_ep_rank=[32741, 32770, 32669, 32886, 32946, 32763, 32547, 32822] avg=32768.0000 max=32946 max_rank=4 min=32547 min_rank=6 imbalance_ratio=1.005432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=4 tokens_per_ep_rank=[33078, 32676, 32607, 33015, 32865, 32722, 32632, 32549] avg=32768.0000 max=33078 max_rank=0 min=32549 min_rank=7 imbalance_ratio=1.009460 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=5 tokens_per_ep_rank=[32741, 32770, 32669, 32886, 32946, 32763, 32547, 32822] avg=32768.0000 max=32946 max_rank=4 min=32547 min_rank=6 imbalance_ratio=1.005432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=5 tokens_per_ep_rank=[32741, 32770, 32669, 32886, 32946, 32763, 32547, 32822] avg=32768.0000 max=32946 max_rank=4 min=32547 min_rank=6 imbalance_ratio=1.005432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=5 tokens_per_ep_rank=[32741, 32770, 32669, 32886, 32946, 32763, 32547, 32822] avg=32768.0000 max=32946 max_rank=4 min=32547 min_rank=6 imbalance_ratio=1.005432 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=5 tokens_per_ep_rank=[32741, 32770, 32669, 32886, 32946, 32763, 32547, 32822] avg=32768.0000 max=32946 max_rank=4 min=32547 min_rank=6 imbalance_ratio=1.005432 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=5 tokens_per_ep_rank=[32741, 32770, 32669, 32886, 32946, 32763, 32547, 32822] avg=32768.0000 max=32946 max_rank=4 min=32547 min_rank=6 imbalance_ratio=1.005432 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=5 tokens_per_ep_rank=[32741, 32770, 32669, 32886, 32946, 32763, 32547, 32822] avg=32768.0000 max=32946 max_rank=4 min=32547 min_rank=6 imbalance_ratio=1.005432 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=5 tokens_per_ep_rank=[32741, 32770, 32669, 32886, 32946, 32763, 32547, 32822] avg=32768.0000 max=32946 max_rank=4 min=32547 min_rank=6 imbalance_ratio=1.005432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=5 tokens_per_ep_rank=[32741, 32770, 32669, 32886, 32946, 32763, 32547, 32822] avg=32768.0000 max=32946 max_rank=4 min=32547 min_rank=6 imbalance_ratio=1.005432 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=5 tokens_per_ep_rank=[32741, 32770, 32669, 32886, 32946, 32763, 32547, 32822] avg=32768.0000 max=32946 max_rank=4 min=32547 min_rank=6 imbalance_ratio=1.005432 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=5 tokens_per_ep_rank=[32741, 32770, 32669, 32886, 32946, 32763, 32547, 32822] avg=32768.0000 max=32946 max_rank=4 min=32547 min_rank=6 imbalance_ratio=1.005432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=5 tokens_per_ep_rank=[32741, 32770, 32669, 32886, 32946, 32763, 32547, 32822] avg=32768.0000 max=32946 max_rank=4 min=32547 min_rank=6 imbalance_ratio=1.005432 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=6 tokens_per_ep_rank=[32760, 32816, 33057, 32796, 32437, 32710, 32905, 32663] avg=32768.0000 max=33057 max_rank=2 min=32437 min_rank=4 imbalance_ratio=1.008820 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=6 tokens_per_ep_rank=[32760, 32816, 33057, 32796, 32437, 32710, 32905, 32663] avg=32768.0000 max=33057 max_rank=2 min=32437 min_rank=4 imbalance_ratio=1.008820 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=6 tokens_per_ep_rank=[32760, 32816, 33057, 32796, 32437, 32710, 32905, 32663] avg=32768.0000 max=33057 max_rank=2 min=32437 min_rank=4 imbalance_ratio=1.008820 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=6 tokens_per_ep_rank=[32760, 32816, 33057, 32796, 32437, 32710, 32905, 32663] avg=32768.0000 max=33057 max_rank=2 min=32437 min_rank=4 imbalance_ratio=1.008820 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=5 tokens_per_ep_rank=[32741, 32770, 32669, 32886, 32946, 32763, 32547, 32822] avg=32768.0000 max=32946 max_rank=4 min=32547 min_rank=6 imbalance_ratio=1.005432 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=6 tokens_per_ep_rank=[32760, 32816, 33057, 32796, 32437, 32710, 32905, 32663] avg=32768.0000 max=33057 max_rank=2 min=32437 min_rank=4 imbalance_ratio=1.008820 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=6 tokens_per_ep_rank=[32760, 32816, 33057, 32796, 32437, 32710, 32905, 32663] avg=32768.0000 max=33057 max_rank=2 min=32437 min_rank=4 imbalance_ratio=1.008820 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=6 tokens_per_ep_rank=[32760, 32816, 33057, 32796, 32437, 32710, 32905, 32663] avg=32768.0000 max=33057 max_rank=2 min=32437 min_rank=4 imbalance_ratio=1.008820 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=6 tokens_per_ep_rank=[32760, 32816, 33057, 32796, 32437, 32710, 32905, 32663] avg=32768.0000 max=33057 max_rank=2 min=32437 min_rank=4 imbalance_ratio=1.008820 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=6 tokens_per_ep_rank=[32760, 32816, 33057, 32796, 32437, 32710, 32905, 32663] avg=32768.0000 max=33057 max_rank=2 min=32437 min_rank=4 imbalance_ratio=1.008820 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=6 tokens_per_ep_rank=[32760, 32816, 33057, 32796, 32437, 32710, 32905, 32663] avg=32768.0000 max=33057 max_rank=2 min=32437 min_rank=4 imbalance_ratio=1.008820 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=6 tokens_per_ep_rank=[32760, 32816, 33057, 32796, 32437, 32710, 32905, 32663] avg=32768.0000 max=33057 max_rank=2 min=32437 min_rank=4 imbalance_ratio=1.008820 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=6 tokens_per_ep_rank=[32760, 32816, 33057, 32796, 32437, 32710, 32905, 32663] avg=32768.0000 max=33057 max_rank=2 min=32437 min_rank=4 imbalance_ratio=1.008820 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=6 tokens_per_ep_rank=[32760, 32816, 33057, 32796, 32437, 32710, 32905, 32663] avg=32768.0000 max=33057 max_rank=2 min=32437 min_rank=4 imbalance_ratio=1.008820 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=6 tokens_per_ep_rank=[32760, 32816, 33057, 32796, 32437, 32710, 32905, 32663] avg=32768.0000 max=33057 max_rank=2 min=32437 min_rank=4 imbalance_ratio=1.008820 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=6 tokens_per_ep_rank=[32760, 32816, 33057, 32796, 32437, 32710, 32905, 32663] avg=32768.0000 max=33057 max_rank=2 min=32437 min_rank=4 imbalance_ratio=1.008820 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=7 tokens_per_ep_rank=[32678, 32911, 32767, 32895, 32892, 32745, 32580, 32676] avg=32768.0000 max=32911 max_rank=1 min=32580 min_rank=6 imbalance_ratio=1.004364 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=7 tokens_per_ep_rank=[32678, 32911, 32767, 32895, 32892, 32745, 32580, 32676] avg=32768.0000 max=32911 max_rank=1 min=32580 min_rank=6 imbalance_ratio=1.004364 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=7 tokens_per_ep_rank=[32678, 32911, 32767, 32895, 32892, 32745, 32580, 32676] avg=32768.0000 max=32911 max_rank=1 min=32580 min_rank=6 imbalance_ratio=1.004364 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=7 tokens_per_ep_rank=[32678, 32911, 32767, 32895, 32892, 32745, 32580, 32676] avg=32768.0000 max=32911 max_rank=1 min=32580 min_rank=6 imbalance_ratio=1.004364 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=6 tokens_per_ep_rank=[32760, 32816, 33057, 32796, 32437, 32710, 32905, 32663] avg=32768.0000 max=33057 max_rank=2 min=32437 min_rank=4 imbalance_ratio=1.008820 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=7 tokens_per_ep_rank=[32678, 32911, 32767, 32895, 32892, 32745, 32580, 32676] avg=32768.0000 max=32911 max_rank=1 min=32580 min_rank=6 imbalance_ratio=1.004364 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=7 tokens_per_ep_rank=[32678, 32911, 32767, 32895, 32892, 32745, 32580, 32676] avg=32768.0000 max=32911 max_rank=1 min=32580 min_rank=6 imbalance_ratio=1.004364 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=7 tokens_per_ep_rank=[32678, 32911, 32767, 32895, 32892, 32745, 32580, 32676] avg=32768.0000 max=32911 max_rank=1 min=32580 min_rank=6 imbalance_ratio=1.004364 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=7 tokens_per_ep_rank=[32678, 32911, 32767, 32895, 32892, 32745, 32580, 32676] avg=32768.0000 max=32911 max_rank=1 min=32580 min_rank=6 imbalance_ratio=1.004364 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=7 tokens_per_ep_rank=[32678, 32911, 32767, 32895, 32892, 32745, 32580, 32676] avg=32768.0000 max=32911 max_rank=1 min=32580 min_rank=6 imbalance_ratio=1.004364 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=7 tokens_per_ep_rank=[32678, 32911, 32767, 32895, 32892, 32745, 32580, 32676] avg=32768.0000 max=32911 max_rank=1 min=32580 min_rank=6 imbalance_ratio=1.004364 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=7 tokens_per_ep_rank=[32678, 32911, 32767, 32895, 32892, 32745, 32580, 32676] avg=32768.0000 max=32911 max_rank=1 min=32580 min_rank=6 imbalance_ratio=1.004364 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=7 tokens_per_ep_rank=[32678, 32911, 32767, 32895, 32892, 32745, 32580, 32676] avg=32768.0000 max=32911 max_rank=1 min=32580 min_rank=6 imbalance_ratio=1.004364 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=7 tokens_per_ep_rank=[32678, 32911, 32767, 32895, 32892, 32745, 32580, 32676] avg=32768.0000 max=32911 max_rank=1 min=32580 min_rank=6 imbalance_ratio=1.004364 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=7 tokens_per_ep_rank=[32678, 32911, 32767, 32895, 32892, 32745, 32580, 32676] avg=32768.0000 max=32911 max_rank=1 min=32580 min_rank=6 imbalance_ratio=1.004364 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=7 tokens_per_ep_rank=[32678, 32911, 32767, 32895, 32892, 32745, 32580, 32676] avg=32768.0000 max=32911 max_rank=1 min=32580 min_rank=6 imbalance_ratio=1.004364 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=8 tokens_per_ep_rank=[32701, 33117, 32732, 32730, 32764, 32601, 32820, 32679] avg=32768.0000 max=33117 max_rank=1 min=32601 min_rank=5 imbalance_ratio=1.010651 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=8 tokens_per_ep_rank=[32701, 33117, 32732, 32730, 32764, 32601, 32820, 32679] avg=32768.0000 max=33117 max_rank=1 min=32601 min_rank=5 imbalance_ratio=1.010651 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=8 tokens_per_ep_rank=[32701, 33117, 32732, 32730, 32764, 32601, 32820, 32679] avg=32768.0000 max=33117 max_rank=1 min=32601 min_rank=5 imbalance_ratio=1.010651 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=8 tokens_per_ep_rank=[32701, 33117, 32732, 32730, 32764, 32601, 32820, 32679] avg=32768.0000 max=33117 max_rank=1 min=32601 min_rank=5 imbalance_ratio=1.010651 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=7 tokens_per_ep_rank=[32678, 32911, 32767, 32895, 32892, 32745, 32580, 32676] avg=32768.0000 max=32911 max_rank=1 min=32580 min_rank=6 imbalance_ratio=1.004364 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=8 tokens_per_ep_rank=[32701, 33117, 32732, 32730, 32764, 32601, 32820, 32679] avg=32768.0000 max=33117 max_rank=1 min=32601 min_rank=5 imbalance_ratio=1.010651 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=8 tokens_per_ep_rank=[32701, 33117, 32732, 32730, 32764, 32601, 32820, 32679] avg=32768.0000 max=33117 max_rank=1 min=32601 min_rank=5 imbalance_ratio=1.010651 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=8 tokens_per_ep_rank=[32701, 33096, 32732, 32730, 32764, 32622, 32820, 32679] avg=32768.0000 max=33096 max_rank=1 min=32622 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=21 max_load_reduction_pct=0.000634 moved_to_replicas=21 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=8 tokens_per_ep_rank=[32701, 33096, 32732, 32730, 32764, 32622, 32820, 32679] avg=32768.0000 max=33096 max_rank=1 min=32622 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=21 max_load_reduction_pct=0.000634 moved_to_replicas=21 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=8 tokens_per_ep_rank=[32701, 33096, 32732, 32730, 32764, 32622, 32820, 32679] avg=32768.0000 max=33096 max_rank=1 min=32622 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=21 max_load_reduction_pct=0.000634 moved_to_replicas=21 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=8 tokens_per_ep_rank=[32701, 33117, 32732, 32730, 32764, 32601, 32820, 32679] avg=32768.0000 max=33117 max_rank=1 min=32601 min_rank=5 imbalance_ratio=1.010651 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=8 tokens_per_ep_rank=[32701, 33096, 32732, 32730, 32764, 32622, 32820, 32679] avg=32768.0000 max=33096 max_rank=1 min=32622 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=21 max_load_reduction_pct=0.000634 moved_to_replicas=21 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=8 tokens_per_ep_rank=[32701, 33096, 32732, 32730, 32764, 32622, 32820, 32679] avg=32768.0000 max=33096 max_rank=1 min=32622 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=21 max_load_reduction_pct=0.000634 moved_to_replicas=21 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=8 tokens_per_ep_rank=[32701, 33117, 32732, 32730, 32764, 32601, 32820, 32679] avg=32768.0000 max=33117 max_rank=1 min=32601 min_rank=5 imbalance_ratio=1.010651 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=8 tokens_per_ep_rank=[32701, 33096, 32732, 32730, 32764, 32622, 32820, 32679] avg=32768.0000 max=33096 max_rank=1 min=32622 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=21 max_load_reduction_pct=0.000634 moved_to_replicas=21 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=8 tokens_per_ep_rank=[32701, 33096, 32732, 32730, 32764, 32622, 32820, 32679] avg=32768.0000 max=33096 max_rank=1 min=32622 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=21 max_load_reduction_pct=0.000634 moved_to_replicas=21 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=9 tokens_per_ep_rank=[32807, 32890, 32824, 33033, 32537, 32606, 32647, 32800] avg=32768.0000 max=33033 max_rank=3 min=32537 min_rank=4 imbalance_ratio=1.008087 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=9 tokens_per_ep_rank=[32807, 32890, 32824, 33033, 32537, 32606, 32647, 32800] avg=32768.0000 max=33033 max_rank=3 min=32537 min_rank=4 imbalance_ratio=1.008087 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=9 tokens_per_ep_rank=[32807, 32890, 32824, 33033, 32537, 32606, 32647, 32800] avg=32768.0000 max=33033 max_rank=3 min=32537 min_rank=4 imbalance_ratio=1.008087 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=9 tokens_per_ep_rank=[32807, 32890, 32824, 33033, 32537, 32606, 32647, 32800] avg=32768.0000 max=33033 max_rank=3 min=32537 min_rank=4 imbalance_ratio=1.008087 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=8 tokens_per_ep_rank=[32701, 33096, 32732, 32730, 32764, 32622, 32820, 32679] avg=32768.0000 max=33096 max_rank=1 min=32622 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=21 max_load_reduction_pct=0.000634 moved_to_replicas=21 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=9 tokens_per_ep_rank=[32807, 32890, 32824, 33033, 32537, 32606, 32647, 32800] avg=32768.0000 max=33033 max_rank=3 min=32537 min_rank=4 imbalance_ratio=1.008087 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=9 tokens_per_ep_rank=[32807, 32890, 32824, 33033, 32537, 32606, 32647, 32800] avg=32768.0000 max=33033 max_rank=3 min=32537 min_rank=4 imbalance_ratio=1.008087 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=9 tokens_per_ep_rank=[32807, 32890, 32824, 33033, 32537, 32606, 32647, 32800] avg=32768.0000 max=33033 max_rank=3 min=32537 min_rank=4 imbalance_ratio=1.008087 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=9 tokens_per_ep_rank=[32807, 32890, 32824, 33033, 32537, 32606, 32647, 32800] avg=32768.0000 max=33033 max_rank=3 min=32537 min_rank=4 imbalance_ratio=1.008087 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=9 tokens_per_ep_rank=[32807, 32890, 32824, 33033, 32537, 32606, 32647, 32800] avg=32768.0000 max=33033 max_rank=3 min=32537 min_rank=4 imbalance_ratio=1.008087 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=9 tokens_per_ep_rank=[32807, 32890, 32824, 33033, 32537, 32606, 32647, 32800] avg=32768.0000 max=33033 max_rank=3 min=32537 min_rank=4 imbalance_ratio=1.008087 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=9 tokens_per_ep_rank=[32807, 32890, 32824, 33033, 32537, 32606, 32647, 32800] avg=32768.0000 max=33033 max_rank=3 min=32537 min_rank=4 imbalance_ratio=1.008087 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=9 tokens_per_ep_rank=[32807, 32890, 32824, 33033, 32537, 32606, 32647, 32800] avg=32768.0000 max=33033 max_rank=3 min=32537 min_rank=4 imbalance_ratio=1.008087 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=9 tokens_per_ep_rank=[32807, 32890, 32824, 33033, 32537, 32606, 32647, 32800] avg=32768.0000 max=33033 max_rank=3 min=32537 min_rank=4 imbalance_ratio=1.008087 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=9 tokens_per_ep_rank=[32807, 32890, 32824, 33033, 32537, 32606, 32647, 32800] avg=32768.0000 max=33033 max_rank=3 min=32537 min_rank=4 imbalance_ratio=1.008087 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=10 tokens_per_ep_rank=[32792, 32723, 32818, 32622, 32875, 32684, 32774, 32856] avg=32768.0000 max=32875 max_rank=4 min=32622 min_rank=3 imbalance_ratio=1.003265 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=10 tokens_per_ep_rank=[32792, 32723, 32818, 32622, 32875, 32684, 32774, 32856] avg=32768.0000 max=32875 max_rank=4 min=32622 min_rank=3 imbalance_ratio=1.003265 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=9 tokens_per_ep_rank=[32807, 32890, 32824, 33033, 32537, 32606, 32647, 32800] avg=32768.0000 max=33033 max_rank=3 min=32537 min_rank=4 imbalance_ratio=1.008087 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=10 tokens_per_ep_rank=[32792, 32723, 32818, 32622, 32875, 32684, 32774, 32856] avg=32768.0000 max=32875 max_rank=4 min=32622 min_rank=3 imbalance_ratio=1.003265 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=9 tokens_per_ep_rank=[32807, 32890, 32824, 33033, 32537, 32606, 32647, 32800] avg=32768.0000 max=33033 max_rank=3 min=32537 min_rank=4 imbalance_ratio=1.008087 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=10 tokens_per_ep_rank=[32792, 32723, 32818, 32622, 32875, 32684, 32774, 32856] avg=32768.0000 max=32875 max_rank=4 min=32622 min_rank=3 imbalance_ratio=1.003265 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=10 tokens_per_ep_rank=[32792, 32723, 32818, 32622, 32875, 32684, 32774, 32856] avg=32768.0000 max=32875 max_rank=4 min=32622 min_rank=3 imbalance_ratio=1.003265 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=10 tokens_per_ep_rank=[32792, 32723, 32818, 32622, 32875, 32684, 32774, 32856] avg=32768.0000 max=32875 max_rank=4 min=32622 min_rank=3 imbalance_ratio=1.003265 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=10 tokens_per_ep_rank=[32792, 32723, 32818, 32622, 32875, 32684, 32774, 32856] avg=32768.0000 max=32875 max_rank=4 min=32622 min_rank=3 imbalance_ratio=1.003265 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=10 tokens_per_ep_rank=[32792, 32723, 32818, 32622, 32875, 32684, 32774, 32856] avg=32768.0000 max=32875 max_rank=4 min=32622 min_rank=3 imbalance_ratio=1.003265 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=10 tokens_per_ep_rank=[32792, 32723, 32818, 32622, 32875, 32684, 32774, 32856] avg=32768.0000 max=32875 max_rank=4 min=32622 min_rank=3 imbalance_ratio=1.003265 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=10 tokens_per_ep_rank=[32792, 32723, 32818, 32622, 32875, 32684, 32774, 32856] avg=32768.0000 max=32875 max_rank=4 min=32622 min_rank=3 imbalance_ratio=1.003265 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=10 tokens_per_ep_rank=[32792, 32723, 32818, 32622, 32875, 32684, 32774, 32856] avg=32768.0000 max=32875 max_rank=4 min=32622 min_rank=3 imbalance_ratio=1.003265 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=10 tokens_per_ep_rank=[32792, 32723, 32818, 32622, 32875, 32684, 32774, 32856] avg=32768.0000 max=32875 max_rank=4 min=32622 min_rank=3 imbalance_ratio=1.003265 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=10 tokens_per_ep_rank=[32792, 32723, 32818, 32622, 32875, 32684, 32774, 32856] avg=32768.0000 max=32875 max_rank=4 min=32622 min_rank=3 imbalance_ratio=1.003265 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=10 tokens_per_ep_rank=[32792, 32723, 32818, 32622, 32875, 32684, 32774, 32856] avg=32768.0000 max=32875 max_rank=4 min=32622 min_rank=3 imbalance_ratio=1.003265 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=10 tokens_per_ep_rank=[32792, 32723, 32818, 32622, 32875, 32684, 32774, 32856] avg=32768.0000 max=32875 max_rank=4 min=32622 min_rank=3 imbalance_ratio=1.003265 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=10 tokens_per_ep_rank=[32792, 32723, 32818, 32622, 32875, 32684, 32774, 32856] avg=32768.0000 max=32875 max_rank=4 min=32622 min_rank=3 imbalance_ratio=1.003265 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=5 records=10 total_expected_recv=326844 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=77696 peak_demand=32856 required_capacity=1.002686 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=5 records=10 total_expected_recv=327410 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=77312 peak_demand=33035 required_capacity=1.008148 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=5 records=10 total_expected_recv=327673 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76928 peak_demand=33057 required_capacity=1.008820 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=5 records=10 total_expected_recv=329017 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=75904 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=5 records=10 total_expected_recv=327536 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=77184 peak_demand=32946 required_capacity=1.005432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=5 records=10 total_expected_recv=328025 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76544 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=5 records=10 total_expected_recv=328029 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76928 peak_demand=33078 required_capacity=1.009460 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=5 records=10 total_expected_recv=326906 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=77824 peak_demand=32905 required_capacity=1.004181 + [2026-08-26 15:53:12] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 213.7 | throughput per GPU (TFLOP/s/GPU): 577.8 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 1.029102E+01 | loss scale: 1.0 | grad norm: 8.066 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 33664 | min safe capacity: 1.0273 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=0 calls=1 budget=40960 expected_recv=32609 expected_overflow=0 exact_recv_per_rank=[32712, 32804, 33010, 32622, 32712, 32609, 32904, 32771] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=0 calls=1 budget=40960 expected_recv=32712 expected_overflow=0 exact_recv_per_rank=[32712, 32804, 33010, 32622, 32712, 32609, 32904, 32771] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=0 calls=1 budget=40960 expected_recv=33010 expected_overflow=0 exact_recv_per_rank=[32712, 32804, 33010, 32622, 32712, 32609, 32904, 32771] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=0 calls=1 budget=40960 expected_recv=32622 expected_overflow=0 exact_recv_per_rank=[32712, 32804, 33010, 32622, 32712, 32609, 32904, 32771] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=0 calls=1 budget=40960 expected_recv=32771 expected_overflow=0 exact_recv_per_rank=[32712, 32804, 33010, 32622, 32712, 32609, 32904, 32771] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=0 calls=1 budget=40960 expected_recv=32904 expected_overflow=0 exact_recv_per_rank=[32712, 32804, 33010, 32622, 32712, 32609, 32904, 32771] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=0 calls=1 budget=40960 expected_recv=32609 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=0 calls=1 budget=40960 expected_recv=32712 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=0 calls=1 budget=40960 expected_recv=32804 expected_overflow=0 exact_recv_per_rank=[32712, 32804, 33010, 32622, 32712, 32609, 32904, 32771] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=0 calls=1 budget=40960 expected_recv=33010 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=0 calls=1 budget=40960 expected_recv=32622 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=0 calls=1 budget=40960 expected_recv=32771 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=0 calls=1 budget=40960 expected_recv=32904 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=0 calls=1 budget=40960 expected_recv=32804 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=0 calls=1 budget=40960 expected_recv=32712 expected_overflow=0 exact_recv_per_rank=[32712, 32804, 33010, 32622, 32712, 32609, 32904, 32771] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=1 calls=1 budget=40960 expected_recv=32718 expected_overflow=0 exact_recv_per_rank=[32698, 32940, 32881, 32935, 32440, 32718, 32977, 32555] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=1 calls=1 budget=40960 expected_recv=32440 expected_overflow=0 exact_recv_per_rank=[32698, 32940, 32881, 32935, 32440, 32718, 32977, 32555] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=1 calls=1 budget=40960 expected_recv=32718 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=1 calls=1 budget=40960 expected_recv=32881 expected_overflow=0 exact_recv_per_rank=[32698, 32940, 32881, 32935, 32440, 32718, 32977, 32555] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=1 calls=1 budget=40960 expected_recv=32555 expected_overflow=0 exact_recv_per_rank=[32698, 32940, 32881, 32935, 32440, 32718, 32977, 32555] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=1 calls=1 budget=40960 expected_recv=32935 expected_overflow=0 exact_recv_per_rank=[32698, 32940, 32881, 32935, 32440, 32718, 32977, 32555] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=0 calls=1 budget=40960 expected_recv=32712 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=1 calls=1 budget=40960 expected_recv=32440 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=1 calls=1 budget=40960 expected_recv=32977 expected_overflow=0 exact_recv_per_rank=[32698, 32940, 32881, 32935, 32440, 32718, 32977, 32555] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=1 calls=1 budget=40960 expected_recv=32881 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=1 calls=1 budget=40960 expected_recv=32555 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=1 calls=1 budget=40960 expected_recv=32935 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=1 calls=1 budget=40960 expected_recv=32940 expected_overflow=0 exact_recv_per_rank=[32698, 32940, 32881, 32935, 32440, 32718, 32977, 32555] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=1 calls=1 budget=40960 expected_recv=32977 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=1 calls=1 budget=40960 expected_recv=32940 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=2 calls=1 budget=40960 expected_recv=32923 expected_overflow=0 exact_recv_per_rank=[32811, 32726, 32573, 32781, 32584, 32923, 32889, 32857] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=2 calls=1 budget=40960 expected_recv=32584 expected_overflow=0 exact_recv_per_rank=[32811, 32726, 32573, 32781, 32584, 32923, 32889, 32857] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=2 calls=1 budget=40960 expected_recv=32923 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=2 calls=1 budget=40960 expected_recv=32573 expected_overflow=0 exact_recv_per_rank=[32811, 32726, 32573, 32781, 32584, 32923, 32889, 32857] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=1 calls=1 budget=40960 expected_recv=32698 expected_overflow=0 exact_recv_per_rank=[32698, 32940, 32881, 32935, 32440, 32718, 32977, 32555] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=2 calls=1 budget=40960 expected_recv=32781 expected_overflow=0 exact_recv_per_rank=[32811, 32726, 32573, 32781, 32584, 32923, 32889, 32857] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=2 calls=1 budget=40960 expected_recv=32857 expected_overflow=0 exact_recv_per_rank=[32811, 32726, 32573, 32781, 32584, 32923, 32889, 32857] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=2 calls=1 budget=40960 expected_recv=32584 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=2 calls=1 budget=40960 expected_recv=32889 expected_overflow=0 exact_recv_per_rank=[32811, 32726, 32573, 32781, 32584, 32923, 32889, 32857] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=2 calls=1 budget=40960 expected_recv=32573 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=1 calls=1 budget=40960 expected_recv=32698 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=2 calls=1 budget=40960 expected_recv=32726 expected_overflow=0 exact_recv_per_rank=[32811, 32726, 32573, 32781, 32584, 32923, 32889, 32857] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=2 calls=1 budget=40960 expected_recv=32781 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=2 calls=1 budget=40960 expected_recv=32857 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=2 calls=1 budget=40960 expected_recv=32889 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=2 calls=1 budget=40960 expected_recv=32726 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=3 calls=1 budget=40960 expected_recv=32957 expected_overflow=0 exact_recv_per_rank=[32873, 32338, 32811, 32719, 32935, 32957, 32749, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=3 calls=1 budget=40960 expected_recv=32935 expected_overflow=0 exact_recv_per_rank=[32873, 32338, 32811, 32719, 32935, 32957, 32749, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=3 calls=1 budget=40960 expected_recv=32957 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=2 calls=1 budget=40960 expected_recv=32811 expected_overflow=0 exact_recv_per_rank=[32811, 32726, 32573, 32781, 32584, 32923, 32889, 32857] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=3 calls=1 budget=40960 expected_recv=32811 expected_overflow=0 exact_recv_per_rank=[32873, 32338, 32811, 32719, 32935, 32957, 32749, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=3 calls=1 budget=40960 expected_recv=32935 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=2 calls=1 budget=40960 expected_recv=32811 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=3 calls=1 budget=40960 expected_recv=32811 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=3 calls=1 budget=40960 expected_recv=32719 expected_overflow=0 exact_recv_per_rank=[32873, 32338, 32811, 32719, 32935, 32957, 32749, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=3 calls=1 budget=40960 expected_recv=32762 expected_overflow=0 exact_recv_per_rank=[32873, 32338, 32811, 32719, 32935, 32957, 32749, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=3 calls=1 budget=40960 expected_recv=32749 expected_overflow=0 exact_recv_per_rank=[32873, 32338, 32811, 32719, 32935, 32957, 32749, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=3 calls=1 budget=40960 expected_recv=32338 expected_overflow=0 exact_recv_per_rank=[32873, 32338, 32811, 32719, 32935, 32957, 32749, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=3 calls=1 budget=40960 expected_recv=32719 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=3 calls=1 budget=40960 expected_recv=32762 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=3 calls=1 budget=40960 expected_recv=32749 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=3 calls=1 budget=40960 expected_recv=32338 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=4 calls=1 budget=40960 expected_recv=32733 expected_overflow=0 exact_recv_per_rank=[32682, 32643, 33096, 32613, 32766, 32733, 32697, 32914] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=4 calls=1 budget=40960 expected_recv=32733 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=4 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32682, 32643, 33096, 32613, 32766, 32733, 32697, 32914] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=3 calls=1 budget=40960 expected_recv=32873 expected_overflow=0 exact_recv_per_rank=[32873, 32338, 32811, 32719, 32935, 32957, 32749, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=4 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32682, 32643, 33096, 32613, 32766, 32733, 32697, 32914] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=3 calls=1 budget=40960 expected_recv=32873 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=4 calls=1 budget=40960 expected_recv=32613 expected_overflow=0 exact_recv_per_rank=[32682, 32643, 33096, 32613, 32766, 32733, 32697, 32914] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=4 calls=1 budget=40960 expected_recv=32914 expected_overflow=0 exact_recv_per_rank=[32682, 32643, 33096, 32613, 32766, 32733, 32697, 32914] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=4 calls=1 budget=40960 expected_recv=32643 expected_overflow=0 exact_recv_per_rank=[32682, 32643, 33096, 32613, 32766, 32733, 32697, 32914] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=4 calls=1 budget=40960 expected_recv=32697 expected_overflow=0 exact_recv_per_rank=[32682, 32643, 33096, 32613, 32766, 32733, 32697, 32914] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=4 calls=1 budget=40960 expected_recv=32613 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=4 calls=1 budget=40960 expected_recv=32914 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=5 calls=1 budget=40960 expected_recv=32623 expected_overflow=0 exact_recv_per_rank=[32842, 32840, 32459, 32815, 32810, 32623, 32745, 33010] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=4 calls=1 budget=40960 expected_recv=32643 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=4 calls=1 budget=40960 expected_recv=32697 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=5 calls=1 budget=40960 expected_recv=32623 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=5 calls=1 budget=40960 expected_recv=32810 expected_overflow=0 exact_recv_per_rank=[32842, 32840, 32459, 32815, 32810, 32623, 32745, 33010] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=5 calls=1 budget=40960 expected_recv=32810 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=4 calls=1 budget=40960 expected_recv=32682 expected_overflow=0 exact_recv_per_rank=[32682, 32643, 33096, 32613, 32766, 32733, 32697, 32914] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=5 calls=1 budget=40960 expected_recv=32459 expected_overflow=0 exact_recv_per_rank=[32842, 32840, 32459, 32815, 32810, 32623, 32745, 33010] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=4 calls=1 budget=40960 expected_recv=32682 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=5 calls=1 budget=40960 expected_recv=32815 expected_overflow=0 exact_recv_per_rank=[32842, 32840, 32459, 32815, 32810, 32623, 32745, 33010] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=5 calls=1 budget=40960 expected_recv=33010 expected_overflow=0 exact_recv_per_rank=[32842, 32840, 32459, 32815, 32810, 32623, 32745, 33010] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=5 calls=1 budget=40960 expected_recv=32459 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=5 calls=1 budget=40960 expected_recv=32840 expected_overflow=0 exact_recv_per_rank=[32842, 32840, 32459, 32815, 32810, 32623, 32745, 33010] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=5 calls=1 budget=40960 expected_recv=32745 expected_overflow=0 exact_recv_per_rank=[32842, 32840, 32459, 32815, 32810, 32623, 32745, 33010] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=5 calls=1 budget=40960 expected_recv=32815 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=5 calls=1 budget=40960 expected_recv=33010 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=6 calls=1 budget=40960 expected_recv=32907 expected_overflow=0 exact_recv_per_rank=[32538, 32761, 32717, 32662, 32676, 32907, 33094, 32789] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=5 calls=1 budget=40960 expected_recv=32840 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=5 calls=1 budget=40960 expected_recv=32745 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=6 calls=1 budget=40960 expected_recv=32907 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=6 calls=1 budget=40960 expected_recv=32676 expected_overflow=0 exact_recv_per_rank=[32538, 32761, 32717, 32662, 32676, 32907, 33094, 32789] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=5 calls=1 budget=40960 expected_recv=32842 expected_overflow=0 exact_recv_per_rank=[32842, 32840, 32459, 32815, 32810, 32623, 32745, 33010] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=6 calls=1 budget=40960 expected_recv=32676 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=6 calls=1 budget=40960 expected_recv=32717 expected_overflow=0 exact_recv_per_rank=[32538, 32761, 32717, 32662, 32676, 32907, 33094, 32789] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=5 calls=1 budget=40960 expected_recv=32842 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=6 calls=1 budget=40960 expected_recv=32662 expected_overflow=0 exact_recv_per_rank=[32538, 32761, 32717, 32662, 32676, 32907, 33094, 32789] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=6 calls=1 budget=40960 expected_recv=32789 expected_overflow=0 exact_recv_per_rank=[32538, 32761, 32717, 32662, 32676, 32907, 33094, 32789] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=6 calls=1 budget=40960 expected_recv=32717 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=6 calls=1 budget=40960 expected_recv=32761 expected_overflow=0 exact_recv_per_rank=[32538, 32761, 32717, 32662, 32676, 32907, 33094, 32789] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=6 calls=1 budget=40960 expected_recv=32662 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=6 calls=1 budget=40960 expected_recv=32789 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=7 calls=1 budget=40960 expected_recv=33070 expected_overflow=0 exact_recv_per_rank=[32683, 32857, 32705, 32963, 32630, 33070, 32487, 32749] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=6 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[32538, 32761, 32717, 32662, 32676, 32907, 33094, 32789] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=6 calls=1 budget=40960 expected_recv=32761 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=7 calls=1 budget=40960 expected_recv=33070 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=6 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=7 calls=1 budget=40960 expected_recv=32630 expected_overflow=0 exact_recv_per_rank=[32683, 32857, 32705, 32963, 32630, 33070, 32487, 32749] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=7 calls=1 budget=40960 expected_recv=32630 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=6 calls=1 budget=40960 expected_recv=32538 expected_overflow=0 exact_recv_per_rank=[32538, 32761, 32717, 32662, 32676, 32907, 33094, 32789] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=7 calls=1 budget=40960 expected_recv=32705 expected_overflow=0 exact_recv_per_rank=[32683, 32857, 32705, 32963, 32630, 33070, 32487, 32749] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=6 calls=1 budget=40960 expected_recv=32538 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=7 calls=1 budget=40960 expected_recv=32705 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=7 calls=1 budget=40960 expected_recv=32963 expected_overflow=0 exact_recv_per_rank=[32683, 32857, 32705, 32963, 32630, 33070, 32487, 32749] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=7 calls=1 budget=40960 expected_recv=32749 expected_overflow=0 exact_recv_per_rank=[32683, 32857, 32705, 32963, 32630, 33070, 32487, 32749] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=7 calls=1 budget=40960 expected_recv=32857 expected_overflow=0 exact_recv_per_rank=[32683, 32857, 32705, 32963, 32630, 33070, 32487, 32749] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=8 calls=1 budget=40960 expected_recv=32790 expected_overflow=0 exact_recv_per_rank=[32658, 32790, 32829, 32753, 32717, 32790, 33001, 32606] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=7 calls=1 budget=40960 expected_recv=32963 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=7 calls=1 budget=40960 expected_recv=32857 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=7 calls=1 budget=40960 expected_recv=32749 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=7 calls=1 budget=40960 expected_recv=32487 expected_overflow=0 exact_recv_per_rank=[32683, 32857, 32705, 32963, 32630, 33070, 32487, 32749] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=8 calls=1 budget=40960 expected_recv=32790 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=8 calls=1 budget=40960 expected_recv=32717 expected_overflow=0 exact_recv_per_rank=[32658, 32790, 32829, 32753, 32717, 32790, 33001, 32606] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=7 calls=1 budget=40960 expected_recv=32487 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=8 calls=1 budget=40960 expected_recv=32717 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=7 calls=1 budget=40960 expected_recv=32683 expected_overflow=0 exact_recv_per_rank=[32683, 32857, 32705, 32963, 32630, 33070, 32487, 32749] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=8 calls=1 budget=40960 expected_recv=32829 expected_overflow=0 exact_recv_per_rank=[32658, 32790, 32829, 32753, 32717, 32790, 33001, 32606] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=7 calls=1 budget=40960 expected_recv=32683 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=8 calls=1 budget=40960 expected_recv=32829 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=8 calls=1 budget=40960 expected_recv=32753 expected_overflow=0 exact_recv_per_rank=[32658, 32790, 32829, 32753, 32717, 32790, 33001, 32606] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=8 calls=1 budget=40960 expected_recv=32790 expected_overflow=0 exact_recv_per_rank=[32658, 32790, 32829, 32753, 32717, 32790, 33001, 32606] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32941 expected_overflow=0 exact_recv_per_rank=[32907, 32587, 32611, 32780, 32778, 32941, 32817, 32723] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=8 calls=1 budget=40960 expected_recv=32606 expected_overflow=0 exact_recv_per_rank=[32658, 32790, 32829, 32753, 32717, 32790, 33001, 32606] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=8 calls=1 budget=40960 expected_recv=32753 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=8 calls=1 budget=40960 expected_recv=32790 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32941 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=8 calls=1 budget=40960 expected_recv=33001 expected_overflow=0 exact_recv_per_rank=[32658, 32790, 32829, 32753, 32717, 32790, 33001, 32606] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=8 calls=1 budget=40960 expected_recv=32606 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32778 expected_overflow=0 exact_recv_per_rank=[32907, 32587, 32611, 32780, 32778, 32941, 32817, 32723] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=8 calls=1 budget=40960 expected_recv=33001 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32778 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=8 calls=1 budget=40960 expected_recv=32658 expected_overflow=0 exact_recv_per_rank=[32658, 32790, 32829, 32753, 32717, 32790, 33001, 32606] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=8 calls=1 budget=40960 expected_recv=32658 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32611 expected_overflow=0 exact_recv_per_rank=[32907, 32587, 32611, 32780, 32778, 32941, 32817, 32723] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32611 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32780 expected_overflow=0 exact_recv_per_rank=[32907, 32587, 32611, 32780, 32778, 32941, 32817, 32723] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32587 expected_overflow=0 exact_recv_per_rank=[32907, 32587, 32611, 32780, 32778, 32941, 32817, 32723] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32723 expected_overflow=0 exact_recv_per_rank=[32907, 32587, 32611, 32780, 32778, 32941, 32817, 32723] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32780 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=1 tokens_per_ep_rank=[32712, 32804, 33010, 32622, 32712, 32609, 32904, 32771] avg=32768.0000 max=33010 max_rank=2 min=32609 min_rank=5 imbalance_ratio=1.007385 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32587 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32817 expected_overflow=0 exact_recv_per_rank=[32907, 32587, 32611, 32780, 32778, 32941, 32817, 32723] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32723 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32817 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=1 tokens_per_ep_rank=[32712, 32804, 33010, 32622, 32712, 32609, 32904, 32771] avg=32768.0000 max=33010 max_rank=2 min=32609 min_rank=5 imbalance_ratio=1.007385 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32907 expected_overflow=0 exact_recv_per_rank=[32907, 32587, 32611, 32780, 32778, 32941, 32817, 32723] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=1 tokens_per_ep_rank=[32712, 32804, 33010, 32622, 32712, 32609, 32904, 32771] avg=32768.0000 max=33010 max_rank=2 min=32609 min_rank=5 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32907 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=1 tokens_per_ep_rank=[32712, 32804, 33010, 32622, 32712, 32609, 32904, 32771] avg=32768.0000 max=33010 max_rank=2 min=32609 min_rank=5 imbalance_ratio=1.007385 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=1 tokens_per_ep_rank=[32712, 32804, 33010, 32622, 32712, 32609, 32904, 32771] avg=32768.0000 max=33010 max_rank=2 min=32609 min_rank=5 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=1 tokens_per_ep_rank=[32712, 32804, 33010, 32622, 32712, 32609, 32904, 32771] avg=32768.0000 max=33010 max_rank=2 min=32609 min_rank=5 imbalance_ratio=1.007385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=1 tokens_per_ep_rank=[32712, 32804, 33010, 32622, 32712, 32609, 32904, 32771] avg=32768.0000 max=33010 max_rank=2 min=32609 min_rank=5 imbalance_ratio=1.007385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=1 tokens_per_ep_rank=[32712, 32804, 33010, 32622, 32712, 32609, 32904, 32771] avg=32768.0000 max=33010 max_rank=2 min=32609 min_rank=5 imbalance_ratio=1.007385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=2 tokens_per_ep_rank=[32698, 32940, 32881, 32935, 32440, 32718, 32977, 32555] avg=32768.0000 max=32977 max_rank=6 min=32440 min_rank=4 imbalance_ratio=1.006378 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=1 tokens_per_ep_rank=[32712, 32804, 33010, 32622, 32712, 32609, 32904, 32771] avg=32768.0000 max=33010 max_rank=2 min=32609 min_rank=5 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=1 tokens_per_ep_rank=[32712, 32804, 33010, 32622, 32712, 32609, 32904, 32771] avg=32768.0000 max=33010 max_rank=2 min=32609 min_rank=5 imbalance_ratio=1.007385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=1 tokens_per_ep_rank=[32712, 32804, 33010, 32622, 32712, 32609, 32904, 32771] avg=32768.0000 max=33010 max_rank=2 min=32609 min_rank=5 imbalance_ratio=1.007385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=1 tokens_per_ep_rank=[32712, 32804, 33010, 32622, 32712, 32609, 32904, 32771] avg=32768.0000 max=33010 max_rank=2 min=32609 min_rank=5 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=1 tokens_per_ep_rank=[32712, 32804, 33010, 32622, 32712, 32609, 32904, 32771] avg=32768.0000 max=33010 max_rank=2 min=32609 min_rank=5 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=1 tokens_per_ep_rank=[32712, 32804, 33010, 32622, 32712, 32609, 32904, 32771] avg=32768.0000 max=33010 max_rank=2 min=32609 min_rank=5 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=2 tokens_per_ep_rank=[32698, 32940, 32881, 32935, 32440, 32718, 32977, 32555] avg=32768.0000 max=32977 max_rank=6 min=32440 min_rank=4 imbalance_ratio=1.006378 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=2 tokens_per_ep_rank=[32698, 32940, 32881, 32935, 32440, 32718, 32977, 32555] avg=32768.0000 max=32977 max_rank=6 min=32440 min_rank=4 imbalance_ratio=1.006378 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=1 tokens_per_ep_rank=[32712, 32804, 33010, 32622, 32712, 32609, 32904, 32771] avg=32768.0000 max=33010 max_rank=2 min=32609 min_rank=5 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=2 tokens_per_ep_rank=[32698, 32940, 32881, 32935, 32440, 32718, 32977, 32555] avg=32768.0000 max=32977 max_rank=6 min=32440 min_rank=4 imbalance_ratio=1.006378 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=1 tokens_per_ep_rank=[32712, 32804, 33010, 32622, 32712, 32609, 32904, 32771] avg=32768.0000 max=33010 max_rank=2 min=32609 min_rank=5 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=2 tokens_per_ep_rank=[32698, 32940, 32881, 32935, 32440, 32718, 32977, 32555] avg=32768.0000 max=32977 max_rank=6 min=32440 min_rank=4 imbalance_ratio=1.006378 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=2 tokens_per_ep_rank=[32698, 32940, 32881, 32935, 32440, 32718, 32977, 32555] avg=32768.0000 max=32977 max_rank=6 min=32440 min_rank=4 imbalance_ratio=1.006378 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=2 tokens_per_ep_rank=[32698, 32940, 32881, 32935, 32440, 32718, 32977, 32555] avg=32768.0000 max=32977 max_rank=6 min=32440 min_rank=4 imbalance_ratio=1.006378 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=3 tokens_per_ep_rank=[32811, 32726, 32573, 32781, 32584, 32923, 32889, 32857] avg=32768.0000 max=32923 max_rank=5 min=32573 min_rank=2 imbalance_ratio=1.004730 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=2 tokens_per_ep_rank=[32698, 32940, 32881, 32935, 32440, 32718, 32977, 32555] avg=32768.0000 max=32977 max_rank=6 min=32440 min_rank=4 imbalance_ratio=1.006378 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=2 tokens_per_ep_rank=[32698, 32940, 32881, 32935, 32440, 32718, 32977, 32555] avg=32768.0000 max=32977 max_rank=6 min=32440 min_rank=4 imbalance_ratio=1.006378 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=2 tokens_per_ep_rank=[32698, 32940, 32881, 32935, 32440, 32718, 32977, 32555] avg=32768.0000 max=32977 max_rank=6 min=32440 min_rank=4 imbalance_ratio=1.006378 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=2 tokens_per_ep_rank=[32698, 32940, 32881, 32935, 32440, 32718, 32977, 32555] avg=32768.0000 max=32977 max_rank=6 min=32440 min_rank=4 imbalance_ratio=1.006378 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=2 tokens_per_ep_rank=[32698, 32940, 32881, 32935, 32440, 32718, 32977, 32555] avg=32768.0000 max=32977 max_rank=6 min=32440 min_rank=4 imbalance_ratio=1.006378 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=2 tokens_per_ep_rank=[32698, 32940, 32881, 32935, 32440, 32718, 32977, 32555] avg=32768.0000 max=32977 max_rank=6 min=32440 min_rank=4 imbalance_ratio=1.006378 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=3 tokens_per_ep_rank=[32811, 32726, 32573, 32781, 32584, 32923, 32889, 32857] avg=32768.0000 max=32923 max_rank=5 min=32573 min_rank=2 imbalance_ratio=1.004730 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=3 tokens_per_ep_rank=[32811, 32726, 32573, 32781, 32584, 32923, 32889, 32857] avg=32768.0000 max=32923 max_rank=5 min=32573 min_rank=2 imbalance_ratio=1.004730 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=2 tokens_per_ep_rank=[32698, 32940, 32881, 32935, 32440, 32718, 32977, 32555] avg=32768.0000 max=32977 max_rank=6 min=32440 min_rank=4 imbalance_ratio=1.006378 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=2 tokens_per_ep_rank=[32698, 32940, 32881, 32935, 32440, 32718, 32977, 32555] avg=32768.0000 max=32977 max_rank=6 min=32440 min_rank=4 imbalance_ratio=1.006378 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=3 tokens_per_ep_rank=[32811, 32726, 32573, 32781, 32584, 32923, 32889, 32857] avg=32768.0000 max=32923 max_rank=5 min=32573 min_rank=2 imbalance_ratio=1.004730 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=2 tokens_per_ep_rank=[32698, 32940, 32881, 32935, 32440, 32718, 32977, 32555] avg=32768.0000 max=32977 max_rank=6 min=32440 min_rank=4 imbalance_ratio=1.006378 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=3 tokens_per_ep_rank=[32811, 32726, 32573, 32781, 32584, 32923, 32889, 32857] avg=32768.0000 max=32923 max_rank=5 min=32573 min_rank=2 imbalance_ratio=1.004730 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=3 tokens_per_ep_rank=[32811, 32726, 32573, 32781, 32584, 32923, 32889, 32857] avg=32768.0000 max=32923 max_rank=5 min=32573 min_rank=2 imbalance_ratio=1.004730 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=4 tokens_per_ep_rank=[32873, 32338, 32811, 32719, 32935, 32957, 32749, 32762] avg=32768.0000 max=32957 max_rank=5 min=32338 min_rank=1 imbalance_ratio=1.005768 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=3 tokens_per_ep_rank=[32811, 32726, 32573, 32781, 32584, 32923, 32889, 32857] avg=32768.0000 max=32923 max_rank=5 min=32573 min_rank=2 imbalance_ratio=1.004730 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=3 tokens_per_ep_rank=[32811, 32726, 32573, 32781, 32584, 32923, 32889, 32857] avg=32768.0000 max=32923 max_rank=5 min=32573 min_rank=2 imbalance_ratio=1.004730 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=3 tokens_per_ep_rank=[32811, 32726, 32573, 32781, 32584, 32923, 32889, 32857] avg=32768.0000 max=32923 max_rank=5 min=32573 min_rank=2 imbalance_ratio=1.004730 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=3 tokens_per_ep_rank=[32811, 32726, 32573, 32781, 32584, 32923, 32889, 32857] avg=32768.0000 max=32923 max_rank=5 min=32573 min_rank=2 imbalance_ratio=1.004730 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=3 tokens_per_ep_rank=[32811, 32726, 32573, 32781, 32584, 32923, 32889, 32857] avg=32768.0000 max=32923 max_rank=5 min=32573 min_rank=2 imbalance_ratio=1.004730 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=3 tokens_per_ep_rank=[32811, 32726, 32573, 32781, 32584, 32923, 32889, 32857] avg=32768.0000 max=32923 max_rank=5 min=32573 min_rank=2 imbalance_ratio=1.004730 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=4 tokens_per_ep_rank=[32873, 32338, 32811, 32719, 32935, 32957, 32749, 32762] avg=32768.0000 max=32957 max_rank=5 min=32338 min_rank=1 imbalance_ratio=1.005768 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=3 tokens_per_ep_rank=[32811, 32726, 32573, 32781, 32584, 32923, 32889, 32857] avg=32768.0000 max=32923 max_rank=5 min=32573 min_rank=2 imbalance_ratio=1.004730 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=4 tokens_per_ep_rank=[32873, 32338, 32811, 32719, 32935, 32957, 32749, 32762] avg=32768.0000 max=32957 max_rank=5 min=32338 min_rank=1 imbalance_ratio=1.005768 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=3 tokens_per_ep_rank=[32811, 32726, 32573, 32781, 32584, 32923, 32889, 32857] avg=32768.0000 max=32923 max_rank=5 min=32573 min_rank=2 imbalance_ratio=1.004730 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=3 tokens_per_ep_rank=[32811, 32726, 32573, 32781, 32584, 32923, 32889, 32857] avg=32768.0000 max=32923 max_rank=5 min=32573 min_rank=2 imbalance_ratio=1.004730 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=4 tokens_per_ep_rank=[32873, 32338, 32811, 32719, 32935, 32957, 32749, 32762] avg=32768.0000 max=32957 max_rank=5 min=32338 min_rank=1 imbalance_ratio=1.005768 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=3 tokens_per_ep_rank=[32811, 32726, 32573, 32781, 32584, 32923, 32889, 32857] avg=32768.0000 max=32923 max_rank=5 min=32573 min_rank=2 imbalance_ratio=1.004730 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=4 tokens_per_ep_rank=[32873, 32338, 32811, 32719, 32935, 32957, 32749, 32762] avg=32768.0000 max=32957 max_rank=5 min=32338 min_rank=1 imbalance_ratio=1.005768 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=5 tokens_per_ep_rank=[32682, 32643, 33117, 32592, 32766, 32733, 32697, 32914] avg=32768.0000 max=33117 max_rank=2 min=32592 min_rank=3 imbalance_ratio=1.010651 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=4 tokens_per_ep_rank=[32873, 32338, 32811, 32719, 32935, 32957, 32749, 32762] avg=32768.0000 max=32957 max_rank=5 min=32338 min_rank=1 imbalance_ratio=1.005768 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=4 tokens_per_ep_rank=[32873, 32338, 32811, 32719, 32935, 32957, 32749, 32762] avg=32768.0000 max=32957 max_rank=5 min=32338 min_rank=1 imbalance_ratio=1.005768 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=4 tokens_per_ep_rank=[32873, 32338, 32811, 32719, 32935, 32957, 32749, 32762] avg=32768.0000 max=32957 max_rank=5 min=32338 min_rank=1 imbalance_ratio=1.005768 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=4 tokens_per_ep_rank=[32873, 32338, 32811, 32719, 32935, 32957, 32749, 32762] avg=32768.0000 max=32957 max_rank=5 min=32338 min_rank=1 imbalance_ratio=1.005768 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=4 tokens_per_ep_rank=[32873, 32338, 32811, 32719, 32935, 32957, 32749, 32762] avg=32768.0000 max=32957 max_rank=5 min=32338 min_rank=1 imbalance_ratio=1.005768 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=5 tokens_per_ep_rank=[32682, 32643, 33096, 32613, 32766, 32733, 32697, 32914] avg=32768.0000 max=33096 max_rank=2 min=32613 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=21 max_load_reduction_pct=0.000634 moved_to_replicas=21 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=4 tokens_per_ep_rank=[32873, 32338, 32811, 32719, 32935, 32957, 32749, 32762] avg=32768.0000 max=32957 max_rank=5 min=32338 min_rank=1 imbalance_ratio=1.005768 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=4 tokens_per_ep_rank=[32873, 32338, 32811, 32719, 32935, 32957, 32749, 32762] avg=32768.0000 max=32957 max_rank=5 min=32338 min_rank=1 imbalance_ratio=1.005768 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=4 tokens_per_ep_rank=[32873, 32338, 32811, 32719, 32935, 32957, 32749, 32762] avg=32768.0000 max=32957 max_rank=5 min=32338 min_rank=1 imbalance_ratio=1.005768 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=5 tokens_per_ep_rank=[32682, 32643, 33117, 32592, 32766, 32733, 32697, 32914] avg=32768.0000 max=33117 max_rank=2 min=32592 min_rank=3 imbalance_ratio=1.010651 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=4 tokens_per_ep_rank=[32873, 32338, 32811, 32719, 32935, 32957, 32749, 32762] avg=32768.0000 max=32957 max_rank=5 min=32338 min_rank=1 imbalance_ratio=1.005768 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=5 tokens_per_ep_rank=[32682, 32643, 33117, 32592, 32766, 32733, 32697, 32914] avg=32768.0000 max=33117 max_rank=2 min=32592 min_rank=3 imbalance_ratio=1.010651 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=4 tokens_per_ep_rank=[32873, 32338, 32811, 32719, 32935, 32957, 32749, 32762] avg=32768.0000 max=32957 max_rank=5 min=32338 min_rank=1 imbalance_ratio=1.005768 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=4 tokens_per_ep_rank=[32873, 32338, 32811, 32719, 32935, 32957, 32749, 32762] avg=32768.0000 max=32957 max_rank=5 min=32338 min_rank=1 imbalance_ratio=1.005768 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=6 tokens_per_ep_rank=[32842, 32840, 32459, 32815, 32810, 32623, 32745, 33010] avg=32768.0000 max=33010 max_rank=7 min=32459 min_rank=2 imbalance_ratio=1.007385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=5 tokens_per_ep_rank=[32682, 32643, 33096, 32613, 32766, 32733, 32697, 32914] avg=32768.0000 max=33096 max_rank=2 min=32613 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=21 max_load_reduction_pct=0.000634 moved_to_replicas=21 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=5 tokens_per_ep_rank=[32682, 32643, 33117, 32592, 32766, 32733, 32697, 32914] avg=32768.0000 max=33117 max_rank=2 min=32592 min_rank=3 imbalance_ratio=1.010651 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=5 tokens_per_ep_rank=[32682, 32643, 33096, 32613, 32766, 32733, 32697, 32914] avg=32768.0000 max=33096 max_rank=2 min=32613 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=21 max_load_reduction_pct=0.000634 moved_to_replicas=21 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=5 tokens_per_ep_rank=[32682, 32643, 33117, 32592, 32766, 32733, 32697, 32914] avg=32768.0000 max=33117 max_rank=2 min=32592 min_rank=3 imbalance_ratio=1.010651 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=5 tokens_per_ep_rank=[32682, 32643, 33117, 32592, 32766, 32733, 32697, 32914] avg=32768.0000 max=33117 max_rank=2 min=32592 min_rank=3 imbalance_ratio=1.010651 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=6 tokens_per_ep_rank=[32842, 32840, 32459, 32815, 32810, 32623, 32745, 33010] avg=32768.0000 max=33010 max_rank=7 min=32459 min_rank=2 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=5 tokens_per_ep_rank=[32682, 32643, 33117, 32592, 32766, 32733, 32697, 32914] avg=32768.0000 max=33117 max_rank=2 min=32592 min_rank=3 imbalance_ratio=1.010651 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=5 tokens_per_ep_rank=[32682, 32643, 33117, 32592, 32766, 32733, 32697, 32914] avg=32768.0000 max=33117 max_rank=2 min=32592 min_rank=3 imbalance_ratio=1.010651 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=5 tokens_per_ep_rank=[32682, 32643, 33096, 32613, 32766, 32733, 32697, 32914] avg=32768.0000 max=33096 max_rank=2 min=32613 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=21 max_load_reduction_pct=0.000634 moved_to_replicas=21 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=6 tokens_per_ep_rank=[32842, 32840, 32459, 32815, 32810, 32623, 32745, 33010] avg=32768.0000 max=33010 max_rank=7 min=32459 min_rank=2 imbalance_ratio=1.007385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=5 tokens_per_ep_rank=[32682, 32643, 33096, 32613, 32766, 32733, 32697, 32914] avg=32768.0000 max=33096 max_rank=2 min=32613 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=21 max_load_reduction_pct=0.000634 moved_to_replicas=21 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=5 tokens_per_ep_rank=[32682, 32643, 33096, 32613, 32766, 32733, 32697, 32914] avg=32768.0000 max=33096 max_rank=2 min=32613 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=21 max_load_reduction_pct=0.000634 moved_to_replicas=21 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=6 tokens_per_ep_rank=[32842, 32840, 32459, 32815, 32810, 32623, 32745, 33010] avg=32768.0000 max=33010 max_rank=7 min=32459 min_rank=2 imbalance_ratio=1.007385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=5 tokens_per_ep_rank=[32682, 32643, 33096, 32613, 32766, 32733, 32697, 32914] avg=32768.0000 max=33096 max_rank=2 min=32613 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=21 max_load_reduction_pct=0.000634 moved_to_replicas=21 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=5 tokens_per_ep_rank=[32682, 32643, 33096, 32613, 32766, 32733, 32697, 32914] avg=32768.0000 max=33096 max_rank=2 min=32613 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=21 max_load_reduction_pct=0.000634 moved_to_replicas=21 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=6 tokens_per_ep_rank=[32842, 32840, 32459, 32815, 32810, 32623, 32745, 33010] avg=32768.0000 max=33010 max_rank=7 min=32459 min_rank=2 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=7 tokens_per_ep_rank=[32538, 32761, 32717, 32662, 32676, 32907, 33094, 32789] avg=32768.0000 max=33094 max_rank=6 min=32538 min_rank=0 imbalance_ratio=1.009949 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=6 tokens_per_ep_rank=[32842, 32840, 32459, 32815, 32810, 32623, 32745, 33010] avg=32768.0000 max=33010 max_rank=7 min=32459 min_rank=2 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=6 tokens_per_ep_rank=[32842, 32840, 32459, 32815, 32810, 32623, 32745, 33010] avg=32768.0000 max=33010 max_rank=7 min=32459 min_rank=2 imbalance_ratio=1.007385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=6 tokens_per_ep_rank=[32842, 32840, 32459, 32815, 32810, 32623, 32745, 33010] avg=32768.0000 max=33010 max_rank=7 min=32459 min_rank=2 imbalance_ratio=1.007385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=6 tokens_per_ep_rank=[32842, 32840, 32459, 32815, 32810, 32623, 32745, 33010] avg=32768.0000 max=33010 max_rank=7 min=32459 min_rank=2 imbalance_ratio=1.007385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=7 tokens_per_ep_rank=[32538, 32761, 32717, 32662, 32676, 32907, 33094, 32789] avg=32768.0000 max=33094 max_rank=6 min=32538 min_rank=0 imbalance_ratio=1.009949 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=7 tokens_per_ep_rank=[32538, 32761, 32717, 32662, 32676, 32907, 33094, 32789] avg=32768.0000 max=33094 max_rank=6 min=32538 min_rank=0 imbalance_ratio=1.009949 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=6 tokens_per_ep_rank=[32842, 32840, 32459, 32815, 32810, 32623, 32745, 33010] avg=32768.0000 max=33010 max_rank=7 min=32459 min_rank=2 imbalance_ratio=1.007385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=6 tokens_per_ep_rank=[32842, 32840, 32459, 32815, 32810, 32623, 32745, 33010] avg=32768.0000 max=33010 max_rank=7 min=32459 min_rank=2 imbalance_ratio=1.007385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=6 tokens_per_ep_rank=[32842, 32840, 32459, 32815, 32810, 32623, 32745, 33010] avg=32768.0000 max=33010 max_rank=7 min=32459 min_rank=2 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=6 tokens_per_ep_rank=[32842, 32840, 32459, 32815, 32810, 32623, 32745, 33010] avg=32768.0000 max=33010 max_rank=7 min=32459 min_rank=2 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=6 tokens_per_ep_rank=[32842, 32840, 32459, 32815, 32810, 32623, 32745, 33010] avg=32768.0000 max=33010 max_rank=7 min=32459 min_rank=2 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=7 tokens_per_ep_rank=[32538, 32761, 32717, 32662, 32676, 32907, 33094, 32789] avg=32768.0000 max=33094 max_rank=6 min=32538 min_rank=0 imbalance_ratio=1.009949 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=7 tokens_per_ep_rank=[32538, 32761, 32717, 32662, 32676, 32907, 33094, 32789] avg=32768.0000 max=33094 max_rank=6 min=32538 min_rank=0 imbalance_ratio=1.009949 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=8 tokens_per_ep_rank=[32683, 32857, 32705, 32963, 32630, 33070, 32487, 32749] avg=32768.0000 max=33070 max_rank=5 min=32487 min_rank=6 imbalance_ratio=1.009216 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=6 tokens_per_ep_rank=[32842, 32840, 32459, 32815, 32810, 32623, 32745, 33010] avg=32768.0000 max=33010 max_rank=7 min=32459 min_rank=2 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=6 tokens_per_ep_rank=[32842, 32840, 32459, 32815, 32810, 32623, 32745, 33010] avg=32768.0000 max=33010 max_rank=7 min=32459 min_rank=2 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=7 tokens_per_ep_rank=[32538, 32761, 32717, 32662, 32676, 32907, 33094, 32789] avg=32768.0000 max=33094 max_rank=6 min=32538 min_rank=0 imbalance_ratio=1.009949 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=7 tokens_per_ep_rank=[32538, 32761, 32717, 32662, 32676, 32907, 33094, 32789] avg=32768.0000 max=33094 max_rank=6 min=32538 min_rank=0 imbalance_ratio=1.009949 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=7 tokens_per_ep_rank=[32538, 32761, 32717, 32662, 32676, 32907, 33094, 32789] avg=32768.0000 max=33094 max_rank=6 min=32538 min_rank=0 imbalance_ratio=1.009949 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=7 tokens_per_ep_rank=[32538, 32761, 32717, 32662, 32676, 32907, 33094, 32789] avg=32768.0000 max=33094 max_rank=6 min=32538 min_rank=0 imbalance_ratio=1.009949 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=8 tokens_per_ep_rank=[32683, 32857, 32705, 32963, 32630, 33070, 32487, 32749] avg=32768.0000 max=33070 max_rank=5 min=32487 min_rank=6 imbalance_ratio=1.009216 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=8 tokens_per_ep_rank=[32683, 32857, 32705, 32963, 32630, 33070, 32487, 32749] avg=32768.0000 max=33070 max_rank=5 min=32487 min_rank=6 imbalance_ratio=1.009216 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=7 tokens_per_ep_rank=[32538, 32761, 32717, 32662, 32676, 32907, 33094, 32789] avg=32768.0000 max=33094 max_rank=6 min=32538 min_rank=0 imbalance_ratio=1.009949 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=7 tokens_per_ep_rank=[32538, 32761, 32717, 32662, 32676, 32907, 33094, 32789] avg=32768.0000 max=33094 max_rank=6 min=32538 min_rank=0 imbalance_ratio=1.009949 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=7 tokens_per_ep_rank=[32538, 32761, 32717, 32662, 32676, 32907, 33094, 32789] avg=32768.0000 max=33094 max_rank=6 min=32538 min_rank=0 imbalance_ratio=1.009949 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=7 tokens_per_ep_rank=[32538, 32761, 32717, 32662, 32676, 32907, 33094, 32789] avg=32768.0000 max=33094 max_rank=6 min=32538 min_rank=0 imbalance_ratio=1.009949 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=7 tokens_per_ep_rank=[32538, 32761, 32717, 32662, 32676, 32907, 33094, 32789] avg=32768.0000 max=33094 max_rank=6 min=32538 min_rank=0 imbalance_ratio=1.009949 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=8 tokens_per_ep_rank=[32683, 32857, 32705, 32963, 32630, 33070, 32487, 32749] avg=32768.0000 max=33070 max_rank=5 min=32487 min_rank=6 imbalance_ratio=1.009216 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=8 tokens_per_ep_rank=[32683, 32857, 32705, 32963, 32630, 33070, 32487, 32749] avg=32768.0000 max=33070 max_rank=5 min=32487 min_rank=6 imbalance_ratio=1.009216 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=9 tokens_per_ep_rank=[32658, 32790, 32829, 32753, 32717, 32790, 33001, 32606] avg=32768.0000 max=33001 max_rank=6 min=32606 min_rank=7 imbalance_ratio=1.007111 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=7 tokens_per_ep_rank=[32538, 32761, 32717, 32662, 32676, 32907, 33094, 32789] avg=32768.0000 max=33094 max_rank=6 min=32538 min_rank=0 imbalance_ratio=1.009949 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=7 tokens_per_ep_rank=[32538, 32761, 32717, 32662, 32676, 32907, 33094, 32789] avg=32768.0000 max=33094 max_rank=6 min=32538 min_rank=0 imbalance_ratio=1.009949 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=8 tokens_per_ep_rank=[32683, 32857, 32705, 32963, 32630, 33070, 32487, 32749] avg=32768.0000 max=33070 max_rank=5 min=32487 min_rank=6 imbalance_ratio=1.009216 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=8 tokens_per_ep_rank=[32683, 32857, 32705, 32963, 32630, 33070, 32487, 32749] avg=32768.0000 max=33070 max_rank=5 min=32487 min_rank=6 imbalance_ratio=1.009216 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=9 tokens_per_ep_rank=[32658, 32790, 32829, 32753, 32717, 32790, 33001, 32606] avg=32768.0000 max=33001 max_rank=6 min=32606 min_rank=7 imbalance_ratio=1.007111 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=8 tokens_per_ep_rank=[32683, 32857, 32705, 32963, 32630, 33070, 32487, 32749] avg=32768.0000 max=33070 max_rank=5 min=32487 min_rank=6 imbalance_ratio=1.009216 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=8 tokens_per_ep_rank=[32683, 32857, 32705, 32963, 32630, 33070, 32487, 32749] avg=32768.0000 max=33070 max_rank=5 min=32487 min_rank=6 imbalance_ratio=1.009216 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=9 tokens_per_ep_rank=[32658, 32790, 32829, 32753, 32717, 32790, 33001, 32606] avg=32768.0000 max=33001 max_rank=6 min=32606 min_rank=7 imbalance_ratio=1.007111 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=8 tokens_per_ep_rank=[32683, 32857, 32705, 32963, 32630, 33070, 32487, 32749] avg=32768.0000 max=33070 max_rank=5 min=32487 min_rank=6 imbalance_ratio=1.009216 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=8 tokens_per_ep_rank=[32683, 32857, 32705, 32963, 32630, 33070, 32487, 32749] avg=32768.0000 max=33070 max_rank=5 min=32487 min_rank=6 imbalance_ratio=1.009216 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=8 tokens_per_ep_rank=[32683, 32857, 32705, 32963, 32630, 33070, 32487, 32749] avg=32768.0000 max=33070 max_rank=5 min=32487 min_rank=6 imbalance_ratio=1.009216 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=8 tokens_per_ep_rank=[32683, 32857, 32705, 32963, 32630, 33070, 32487, 32749] avg=32768.0000 max=33070 max_rank=5 min=32487 min_rank=6 imbalance_ratio=1.009216 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=9 tokens_per_ep_rank=[32658, 32790, 32829, 32753, 32717, 32790, 33001, 32606] avg=32768.0000 max=33001 max_rank=6 min=32606 min_rank=7 imbalance_ratio=1.007111 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=8 tokens_per_ep_rank=[32683, 32857, 32705, 32963, 32630, 33070, 32487, 32749] avg=32768.0000 max=33070 max_rank=5 min=32487 min_rank=6 imbalance_ratio=1.009216 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=9 tokens_per_ep_rank=[32658, 32790, 32829, 32753, 32717, 32790, 33001, 32606] avg=32768.0000 max=33001 max_rank=6 min=32606 min_rank=7 imbalance_ratio=1.007111 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=10 tokens_per_ep_rank=[32907, 32587, 32611, 32780, 32778, 32941, 32817, 32723] avg=32768.0000 max=32941 max_rank=5 min=32587 min_rank=1 imbalance_ratio=1.005280 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=8 tokens_per_ep_rank=[32683, 32857, 32705, 32963, 32630, 33070, 32487, 32749] avg=32768.0000 max=33070 max_rank=5 min=32487 min_rank=6 imbalance_ratio=1.009216 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=8 tokens_per_ep_rank=[32683, 32857, 32705, 32963, 32630, 33070, 32487, 32749] avg=32768.0000 max=33070 max_rank=5 min=32487 min_rank=6 imbalance_ratio=1.009216 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=9 tokens_per_ep_rank=[32658, 32790, 32829, 32753, 32717, 32790, 33001, 32606] avg=32768.0000 max=33001 max_rank=6 min=32606 min_rank=7 imbalance_ratio=1.007111 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=10 tokens_per_ep_rank=[32907, 32587, 32611, 32780, 32778, 32941, 32817, 32723] avg=32768.0000 max=32941 max_rank=5 min=32587 min_rank=1 imbalance_ratio=1.005280 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=9 tokens_per_ep_rank=[32658, 32790, 32829, 32753, 32717, 32790, 33001, 32606] avg=32768.0000 max=33001 max_rank=6 min=32606 min_rank=7 imbalance_ratio=1.007111 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=9 tokens_per_ep_rank=[32658, 32790, 32829, 32753, 32717, 32790, 33001, 32606] avg=32768.0000 max=33001 max_rank=6 min=32606 min_rank=7 imbalance_ratio=1.007111 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=9 tokens_per_ep_rank=[32658, 32790, 32829, 32753, 32717, 32790, 33001, 32606] avg=32768.0000 max=33001 max_rank=6 min=32606 min_rank=7 imbalance_ratio=1.007111 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=10 tokens_per_ep_rank=[32907, 32587, 32611, 32780, 32778, 32941, 32817, 32723] avg=32768.0000 max=32941 max_rank=5 min=32587 min_rank=1 imbalance_ratio=1.005280 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=9 tokens_per_ep_rank=[32658, 32790, 32829, 32753, 32717, 32790, 33001, 32606] avg=32768.0000 max=33001 max_rank=6 min=32606 min_rank=7 imbalance_ratio=1.007111 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=9 tokens_per_ep_rank=[32658, 32790, 32829, 32753, 32717, 32790, 33001, 32606] avg=32768.0000 max=33001 max_rank=6 min=32606 min_rank=7 imbalance_ratio=1.007111 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=9 tokens_per_ep_rank=[32658, 32790, 32829, 32753, 32717, 32790, 33001, 32606] avg=32768.0000 max=33001 max_rank=6 min=32606 min_rank=7 imbalance_ratio=1.007111 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=10 tokens_per_ep_rank=[32907, 32587, 32611, 32780, 32778, 32941, 32817, 32723] avg=32768.0000 max=32941 max_rank=5 min=32587 min_rank=1 imbalance_ratio=1.005280 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=9 tokens_per_ep_rank=[32658, 32790, 32829, 32753, 32717, 32790, 33001, 32606] avg=32768.0000 max=33001 max_rank=6 min=32606 min_rank=7 imbalance_ratio=1.007111 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=9 tokens_per_ep_rank=[32658, 32790, 32829, 32753, 32717, 32790, 33001, 32606] avg=32768.0000 max=33001 max_rank=6 min=32606 min_rank=7 imbalance_ratio=1.007111 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=10 tokens_per_ep_rank=[32907, 32587, 32611, 32780, 32778, 32941, 32817, 32723] avg=32768.0000 max=32941 max_rank=5 min=32587 min_rank=1 imbalance_ratio=1.005280 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=9 tokens_per_ep_rank=[32658, 32790, 32829, 32753, 32717, 32790, 33001, 32606] avg=32768.0000 max=33001 max_rank=6 min=32606 min_rank=7 imbalance_ratio=1.007111 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=10 tokens_per_ep_rank=[32907, 32587, 32611, 32780, 32778, 32941, 32817, 32723] avg=32768.0000 max=32941 max_rank=5 min=32587 min_rank=1 imbalance_ratio=1.005280 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=9 tokens_per_ep_rank=[32658, 32790, 32829, 32753, 32717, 32790, 33001, 32606] avg=32768.0000 max=33001 max_rank=6 min=32606 min_rank=7 imbalance_ratio=1.007111 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=10 tokens_per_ep_rank=[32907, 32587, 32611, 32780, 32778, 32941, 32817, 32723] avg=32768.0000 max=32941 max_rank=5 min=32587 min_rank=1 imbalance_ratio=1.005280 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=10 tokens_per_ep_rank=[32907, 32587, 32611, 32780, 32778, 32941, 32817, 32723] avg=32768.0000 max=32941 max_rank=5 min=32587 min_rank=1 imbalance_ratio=1.005280 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=10 tokens_per_ep_rank=[32907, 32587, 32611, 32780, 32778, 32941, 32817, 32723] avg=32768.0000 max=32941 max_rank=5 min=32587 min_rank=1 imbalance_ratio=1.005280 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=10 tokens_per_ep_rank=[32907, 32587, 32611, 32780, 32778, 32941, 32817, 32723] avg=32768.0000 max=32941 max_rank=5 min=32587 min_rank=1 imbalance_ratio=1.005280 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=10 tokens_per_ep_rank=[32907, 32587, 32611, 32780, 32778, 32941, 32817, 32723] avg=32768.0000 max=32941 max_rank=5 min=32587 min_rank=1 imbalance_ratio=1.005280 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=10 tokens_per_ep_rank=[32907, 32587, 32611, 32780, 32778, 32941, 32817, 32723] avg=32768.0000 max=32941 max_rank=5 min=32587 min_rank=1 imbalance_ratio=1.005280 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=10 tokens_per_ep_rank=[32907, 32587, 32611, 32780, 32778, 32941, 32817, 32723] avg=32768.0000 max=32941 max_rank=5 min=32587 min_rank=1 imbalance_ratio=1.005280 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=10 tokens_per_ep_rank=[32907, 32587, 32611, 32780, 32778, 32941, 32817, 32723] avg=32768.0000 max=32941 max_rank=5 min=32587 min_rank=1 imbalance_ratio=1.005280 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=10 tokens_per_ep_rank=[32907, 32587, 32611, 32780, 32778, 32941, 32817, 32723] avg=32768.0000 max=32941 max_rank=5 min=32587 min_rank=1 imbalance_ratio=1.005280 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=10 tokens_per_ep_rank=[32907, 32587, 32611, 32780, 32778, 32941, 32817, 32723] avg=32768.0000 max=32941 max_rank=5 min=32587 min_rank=1 imbalance_ratio=1.005280 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=6 records=10 total_expected_recv=328271 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76800 peak_demand=33070 required_capacity=1.009216 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=6 records=10 total_expected_recv=327048 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=77440 peak_demand=32935 required_capacity=1.005096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=6 records=10 total_expected_recv=327692 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76672 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=6 records=10 total_expected_recv=327736 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76800 peak_demand=33010 required_capacity=1.007385 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=6 records=10 total_expected_recv=327643 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76928 peak_demand=32963 required_capacity=1.005951 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=6 records=10 total_expected_recv=328360 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76416 peak_demand=33094 required_capacity=1.009949 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=6 records=10 total_expected_recv=327404 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76928 peak_demand=32907 required_capacity=1.004242 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=6 records=10 total_expected_recv=327286 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=77696 peak_demand=32940 required_capacity=1.005249 + [2026-08-26 15:53:13] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 212.4 | throughput per GPU (TFLOP/s/GPU): 581.3 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 9.208351E+00 | loss scale: 1.0 | grad norm: 4.959 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 33664 | min safe capacity: 1.0273 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=0 calls=1 budget=40960 expected_recv=32662 expected_overflow=0 exact_recv_per_rank=[32620, 32551, 32662, 32904, 32888, 32705, 32928, 32886] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=0 calls=1 budget=40960 expected_recv=32888 expected_overflow=0 exact_recv_per_rank=[32620, 32551, 32662, 32904, 32888, 32705, 32928, 32886] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=0 calls=1 budget=40960 expected_recv=32904 expected_overflow=0 exact_recv_per_rank=[32620, 32551, 32662, 32904, 32888, 32705, 32928, 32886] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=0 calls=1 budget=40960 expected_recv=32551 expected_overflow=0 exact_recv_per_rank=[32620, 32551, 32662, 32904, 32888, 32705, 32928, 32886] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=0 calls=1 budget=40960 expected_recv=32886 expected_overflow=0 exact_recv_per_rank=[32620, 32551, 32662, 32904, 32888, 32705, 32928, 32886] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=0 calls=1 budget=40960 expected_recv=32928 expected_overflow=0 exact_recv_per_rank=[32620, 32551, 32662, 32904, 32888, 32705, 32928, 32886] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=0 calls=1 budget=40960 expected_recv=32705 expected_overflow=0 exact_recv_per_rank=[32620, 32551, 32662, 32904, 32888, 32705, 32928, 32886] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=0 calls=1 budget=40960 expected_recv=32662 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=0 calls=1 budget=40960 expected_recv=32888 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=0 calls=1 budget=40960 expected_recv=32904 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=0 calls=1 budget=40960 expected_recv=32620 expected_overflow=0 exact_recv_per_rank=[32620, 32551, 32662, 32904, 32888, 32705, 32928, 32886] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=0 calls=1 budget=40960 expected_recv=32551 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=0 calls=1 budget=40960 expected_recv=32886 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=0 calls=1 budget=40960 expected_recv=32928 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=0 calls=1 budget=40960 expected_recv=32705 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=0 calls=1 budget=40960 expected_recv=32620 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=1 calls=1 budget=40960 expected_recv=32893 expected_overflow=0 exact_recv_per_rank=[32721, 32663, 32893, 32675, 32891, 32678, 32933, 32690] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=1 calls=1 budget=40960 expected_recv=32891 expected_overflow=0 exact_recv_per_rank=[32721, 32663, 32893, 32675, 32891, 32678, 32933, 32690] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=1 calls=1 budget=40960 expected_recv=32675 expected_overflow=0 exact_recv_per_rank=[32721, 32663, 32893, 32675, 32891, 32678, 32933, 32690] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=1 calls=1 budget=40960 expected_recv=32893 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=1 calls=1 budget=40960 expected_recv=32690 expected_overflow=0 exact_recv_per_rank=[32721, 32663, 32893, 32675, 32891, 32678, 32933, 32690] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=1 calls=1 budget=40960 expected_recv=32678 expected_overflow=0 exact_recv_per_rank=[32721, 32663, 32893, 32675, 32891, 32678, 32933, 32690] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=1 calls=1 budget=40960 expected_recv=32663 expected_overflow=0 exact_recv_per_rank=[32721, 32663, 32893, 32675, 32891, 32678, 32933, 32690] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=1 calls=1 budget=40960 expected_recv=32933 expected_overflow=0 exact_recv_per_rank=[32721, 32663, 32893, 32675, 32891, 32678, 32933, 32690] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=1 calls=1 budget=40960 expected_recv=32891 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=1 calls=1 budget=40960 expected_recv=32675 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=1 calls=1 budget=40960 expected_recv=32690 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=1 calls=1 budget=40960 expected_recv=32678 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=1 calls=1 budget=40960 expected_recv=32663 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=1 calls=1 budget=40960 expected_recv=32933 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=1 calls=1 budget=40960 expected_recv=32721 expected_overflow=0 exact_recv_per_rank=[32721, 32663, 32893, 32675, 32891, 32678, 32933, 32690] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=1 calls=1 budget=40960 expected_recv=32721 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=2 calls=1 budget=40960 expected_recv=32781 expected_overflow=0 exact_recv_per_rank=[32980, 32753, 32781, 32896, 32597, 32354, 32790, 32993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=2 calls=1 budget=40960 expected_recv=32597 expected_overflow=0 exact_recv_per_rank=[32980, 32753, 32781, 32896, 32597, 32354, 32790, 32993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=2 calls=1 budget=40960 expected_recv=32896 expected_overflow=0 exact_recv_per_rank=[32980, 32753, 32781, 32896, 32597, 32354, 32790, 32993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=2 calls=1 budget=40960 expected_recv=32781 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=2 calls=1 budget=40960 expected_recv=32354 expected_overflow=0 exact_recv_per_rank=[32980, 32753, 32781, 32896, 32597, 32354, 32790, 32993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=2 calls=1 budget=40960 expected_recv=32993 expected_overflow=0 exact_recv_per_rank=[32980, 32753, 32781, 32896, 32597, 32354, 32790, 32993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=2 calls=1 budget=40960 expected_recv=32597 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=2 calls=1 budget=40960 expected_recv=32896 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=2 calls=1 budget=40960 expected_recv=32354 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=2 calls=1 budget=40960 expected_recv=32993 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=2 calls=1 budget=40960 expected_recv=32753 expected_overflow=0 exact_recv_per_rank=[32980, 32753, 32781, 32896, 32597, 32354, 32790, 32993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=2 calls=1 budget=40960 expected_recv=32790 expected_overflow=0 exact_recv_per_rank=[32980, 32753, 32781, 32896, 32597, 32354, 32790, 32993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=2 calls=1 budget=40960 expected_recv=32753 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=2 calls=1 budget=40960 expected_recv=32980 expected_overflow=0 exact_recv_per_rank=[32980, 32753, 32781, 32896, 32597, 32354, 32790, 32993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=2 calls=1 budget=40960 expected_recv=32790 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=2 calls=1 budget=40960 expected_recv=32980 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32827, 32620, 33096, 32743, 32603, 32985, 32679, 32591] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=3 calls=1 budget=40960 expected_recv=32603 expected_overflow=0 exact_recv_per_rank=[32827, 32620, 33096, 32743, 32603, 32985, 32679, 32591] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=3 calls=1 budget=40960 expected_recv=32743 expected_overflow=0 exact_recv_per_rank=[32827, 32620, 33096, 32743, 32603, 32985, 32679, 32591] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=3 calls=1 budget=40960 expected_recv=32603 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=3 calls=1 budget=40960 expected_recv=32985 expected_overflow=0 exact_recv_per_rank=[32827, 32620, 33096, 32743, 32603, 32985, 32679, 32591] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=3 calls=1 budget=40960 expected_recv=32743 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=3 calls=1 budget=40960 expected_recv=32591 expected_overflow=0 exact_recv_per_rank=[32827, 32620, 33096, 32743, 32603, 32985, 32679, 32591] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=3 calls=1 budget=40960 expected_recv=32985 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=3 calls=1 budget=40960 expected_recv=32591 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=3 calls=1 budget=40960 expected_recv=32620 expected_overflow=0 exact_recv_per_rank=[32827, 32620, 33096, 32743, 32603, 32985, 32679, 32591] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=3 calls=1 budget=40960 expected_recv=32679 expected_overflow=0 exact_recv_per_rank=[32827, 32620, 33096, 32743, 32603, 32985, 32679, 32591] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=3 calls=1 budget=40960 expected_recv=32827 expected_overflow=0 exact_recv_per_rank=[32827, 32620, 33096, 32743, 32603, 32985, 32679, 32591] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=3 calls=1 budget=40960 expected_recv=32620 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=3 calls=1 budget=40960 expected_recv=32679 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=3 calls=1 budget=40960 expected_recv=32827 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=4 calls=1 budget=40960 expected_recv=32996 expected_overflow=0 exact_recv_per_rank=[32610, 32906, 32996, 32596, 32525, 32726, 32864, 32921] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=4 calls=1 budget=40960 expected_recv=32525 expected_overflow=0 exact_recv_per_rank=[32610, 32906, 32996, 32596, 32525, 32726, 32864, 32921] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=4 calls=1 budget=40960 expected_recv=32996 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=4 calls=1 budget=40960 expected_recv=32596 expected_overflow=0 exact_recv_per_rank=[32610, 32906, 32996, 32596, 32525, 32726, 32864, 32921] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=4 calls=1 budget=40960 expected_recv=32525 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=4 calls=1 budget=40960 expected_recv=32726 expected_overflow=0 exact_recv_per_rank=[32610, 32906, 32996, 32596, 32525, 32726, 32864, 32921] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=4 calls=1 budget=40960 expected_recv=32596 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=4 calls=1 budget=40960 expected_recv=32921 expected_overflow=0 exact_recv_per_rank=[32610, 32906, 32996, 32596, 32525, 32726, 32864, 32921] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=4 calls=1 budget=40960 expected_recv=32906 expected_overflow=0 exact_recv_per_rank=[32610, 32906, 32996, 32596, 32525, 32726, 32864, 32921] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=4 calls=1 budget=40960 expected_recv=32726 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=4 calls=1 budget=40960 expected_recv=32864 expected_overflow=0 exact_recv_per_rank=[32610, 32906, 32996, 32596, 32525, 32726, 32864, 32921] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=4 calls=1 budget=40960 expected_recv=32921 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=4 calls=1 budget=40960 expected_recv=32906 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=4 calls=1 budget=40960 expected_recv=32864 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=4 calls=1 budget=40960 expected_recv=32610 expected_overflow=0 exact_recv_per_rank=[32610, 32906, 32996, 32596, 32525, 32726, 32864, 32921] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32750 expected_overflow=0 exact_recv_per_rank=[32786, 32971, 32750, 32536, 32869, 32657, 32768, 32807] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=4 calls=1 budget=40960 expected_recv=32610 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32869 expected_overflow=0 exact_recv_per_rank=[32786, 32971, 32750, 32536, 32869, 32657, 32768, 32807] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32750 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32869 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32536 expected_overflow=0 exact_recv_per_rank=[32786, 32971, 32750, 32536, 32869, 32657, 32768, 32807] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32657 expected_overflow=0 exact_recv_per_rank=[32786, 32971, 32750, 32536, 32869, 32657, 32768, 32807] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32807 expected_overflow=0 exact_recv_per_rank=[32786, 32971, 32750, 32536, 32869, 32657, 32768, 32807] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32536 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32971 expected_overflow=0 exact_recv_per_rank=[32786, 32971, 32750, 32536, 32869, 32657, 32768, 32807] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32657 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32807 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32786, 32971, 32750, 32536, 32869, 32657, 32768, 32807] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32971 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32786 expected_overflow=0 exact_recv_per_rank=[32786, 32971, 32750, 32536, 32869, 32657, 32768, 32807] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=6 calls=1 budget=40960 expected_recv=32501 expected_overflow=0 exact_recv_per_rank=[32665, 32625, 32501, 32804, 32917, 33036, 32826, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=6 calls=1 budget=40960 expected_recv=32917 expected_overflow=0 exact_recv_per_rank=[32665, 32625, 32501, 32804, 32917, 33036, 32826, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32786 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=6 calls=1 budget=40960 expected_recv=32804 expected_overflow=0 exact_recv_per_rank=[32665, 32625, 32501, 32804, 32917, 33036, 32826, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=6 calls=1 budget=40960 expected_recv=32501 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=6 calls=1 budget=40960 expected_recv=32917 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=6 calls=1 budget=40960 expected_recv=33036 expected_overflow=0 exact_recv_per_rank=[32665, 32625, 32501, 32804, 32917, 33036, 32826, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=6 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[32665, 32625, 32501, 32804, 32917, 33036, 32826, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=6 calls=1 budget=40960 expected_recv=32804 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=6 calls=1 budget=40960 expected_recv=32625 expected_overflow=0 exact_recv_per_rank=[32665, 32625, 32501, 32804, 32917, 33036, 32826, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=6 calls=1 budget=40960 expected_recv=33036 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=6 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=6 calls=1 budget=40960 expected_recv=32625 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=6 calls=1 budget=40960 expected_recv=32826 expected_overflow=0 exact_recv_per_rank=[32665, 32625, 32501, 32804, 32917, 33036, 32826, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=6 calls=1 budget=40960 expected_recv=32826 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=6 calls=1 budget=40960 expected_recv=32665 expected_overflow=0 exact_recv_per_rank=[32665, 32625, 32501, 32804, 32917, 33036, 32826, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=7 calls=1 budget=40960 expected_recv=32979 expected_overflow=0 exact_recv_per_rank=[32920, 32501, 32979, 33042, 32684, 32815, 32482, 32721] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=7 calls=1 budget=40960 expected_recv=32684 expected_overflow=0 exact_recv_per_rank=[32920, 32501, 32979, 33042, 32684, 32815, 32482, 32721] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=6 calls=1 budget=40960 expected_recv=32665 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=7 calls=1 budget=40960 expected_recv=32979 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=7 calls=1 budget=40960 expected_recv=32815 expected_overflow=0 exact_recv_per_rank=[32920, 32501, 32979, 33042, 32684, 32815, 32482, 32721] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=7 calls=1 budget=40960 expected_recv=33042 expected_overflow=0 exact_recv_per_rank=[32920, 32501, 32979, 33042, 32684, 32815, 32482, 32721] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=7 calls=1 budget=40960 expected_recv=32684 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=7 calls=1 budget=40960 expected_recv=32721 expected_overflow=0 exact_recv_per_rank=[32920, 32501, 32979, 33042, 32684, 32815, 32482, 32721] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=7 calls=1 budget=40960 expected_recv=32815 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=7 calls=1 budget=40960 expected_recv=33042 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=7 calls=1 budget=40960 expected_recv=32501 expected_overflow=0 exact_recv_per_rank=[32920, 32501, 32979, 33042, 32684, 32815, 32482, 32721] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=7 calls=1 budget=40960 expected_recv=32721 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=7 calls=1 budget=40960 expected_recv=32501 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=7 calls=1 budget=40960 expected_recv=32482 expected_overflow=0 exact_recv_per_rank=[32920, 32501, 32979, 33042, 32684, 32815, 32482, 32721] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=7 calls=1 budget=40960 expected_recv=32482 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32596 expected_overflow=0 exact_recv_per_rank=[32785, 32530, 32596, 32795, 32895, 33030, 32984, 32529] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=7 calls=1 budget=40960 expected_recv=32920 expected_overflow=0 exact_recv_per_rank=[32920, 32501, 32979, 33042, 32684, 32815, 32482, 32721] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32895 expected_overflow=0 exact_recv_per_rank=[32785, 32530, 32596, 32795, 32895, 33030, 32984, 32529] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32596 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=7 calls=1 budget=40960 expected_recv=32920 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32895 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=8 calls=1 budget=40960 expected_recv=33030 expected_overflow=0 exact_recv_per_rank=[32785, 32530, 32596, 32795, 32895, 33030, 32984, 32529] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32795 expected_overflow=0 exact_recv_per_rank=[32785, 32530, 32596, 32795, 32895, 33030, 32984, 32529] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32529 expected_overflow=0 exact_recv_per_rank=[32785, 32530, 32596, 32795, 32895, 33030, 32984, 32529] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=8 calls=1 budget=40960 expected_recv=33030 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32795 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32530 expected_overflow=0 exact_recv_per_rank=[32785, 32530, 32596, 32795, 32895, 33030, 32984, 32529] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32529 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32530 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32984 expected_overflow=0 exact_recv_per_rank=[32785, 32530, 32596, 32795, 32895, 33030, 32984, 32529] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32984 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=9 calls=1 budget=40960 expected_recv=32990 expected_overflow=0 exact_recv_per_rank=[32623, 32965, 32990, 32628, 32572, 32642, 32900, 32824] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=9 calls=1 budget=40960 expected_recv=32572 expected_overflow=0 exact_recv_per_rank=[32623, 32965, 32990, 32628, 32572, 32642, 32900, 32824] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32785 expected_overflow=0 exact_recv_per_rank=[32785, 32530, 32596, 32795, 32895, 33030, 32984, 32529] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=9 calls=1 budget=40960 expected_recv=32990 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=9 calls=1 budget=40960 expected_recv=32572 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=9 calls=1 budget=40960 expected_recv=32642 expected_overflow=0 exact_recv_per_rank=[32623, 32965, 32990, 32628, 32572, 32642, 32900, 32824] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32785 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=9 calls=1 budget=40960 expected_recv=32628 expected_overflow=0 exact_recv_per_rank=[32623, 32965, 32990, 32628, 32572, 32642, 32900, 32824] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=9 calls=1 budget=40960 expected_recv=32824 expected_overflow=0 exact_recv_per_rank=[32623, 32965, 32990, 32628, 32572, 32642, 32900, 32824] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=9 calls=1 budget=40960 expected_recv=32965 expected_overflow=0 exact_recv_per_rank=[32623, 32965, 32990, 32628, 32572, 32642, 32900, 32824] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=9 calls=1 budget=40960 expected_recv=32642 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=9 calls=1 budget=40960 expected_recv=32628 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=9 calls=1 budget=40960 expected_recv=32824 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=9 calls=1 budget=40960 expected_recv=32965 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=9 calls=1 budget=40960 expected_recv=32900 expected_overflow=0 exact_recv_per_rank=[32623, 32965, 32990, 32628, 32572, 32642, 32900, 32824] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=9 calls=1 budget=40960 expected_recv=32900 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=1 tokens_per_ep_rank=[32620, 32551, 32662, 32904, 32888, 32705, 32928, 32886] avg=32768.0000 max=32928 max_rank=6 min=32551 min_rank=1 imbalance_ratio=1.004883 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=9 calls=1 budget=40960 expected_recv=32623 expected_overflow=0 exact_recv_per_rank=[32623, 32965, 32990, 32628, 32572, 32642, 32900, 32824] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=1 tokens_per_ep_rank=[32620, 32551, 32662, 32904, 32888, 32705, 32928, 32886] avg=32768.0000 max=32928 max_rank=6 min=32551 min_rank=1 imbalance_ratio=1.004883 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=9 calls=1 budget=40960 expected_recv=32623 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=1 tokens_per_ep_rank=[32620, 32551, 32662, 32904, 32888, 32705, 32928, 32886] avg=32768.0000 max=32928 max_rank=6 min=32551 min_rank=1 imbalance_ratio=1.004883 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=1 tokens_per_ep_rank=[32620, 32551, 32662, 32904, 32888, 32705, 32928, 32886] avg=32768.0000 max=32928 max_rank=6 min=32551 min_rank=1 imbalance_ratio=1.004883 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=1 tokens_per_ep_rank=[32620, 32551, 32662, 32904, 32888, 32705, 32928, 32886] avg=32768.0000 max=32928 max_rank=6 min=32551 min_rank=1 imbalance_ratio=1.004883 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=1 tokens_per_ep_rank=[32620, 32551, 32662, 32904, 32888, 32705, 32928, 32886] avg=32768.0000 max=32928 max_rank=6 min=32551 min_rank=1 imbalance_ratio=1.004883 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=1 tokens_per_ep_rank=[32620, 32551, 32662, 32904, 32888, 32705, 32928, 32886] avg=32768.0000 max=32928 max_rank=6 min=32551 min_rank=1 imbalance_ratio=1.004883 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=1 tokens_per_ep_rank=[32620, 32551, 32662, 32904, 32888, 32705, 32928, 32886] avg=32768.0000 max=32928 max_rank=6 min=32551 min_rank=1 imbalance_ratio=1.004883 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=1 tokens_per_ep_rank=[32620, 32551, 32662, 32904, 32888, 32705, 32928, 32886] avg=32768.0000 max=32928 max_rank=6 min=32551 min_rank=1 imbalance_ratio=1.004883 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=1 tokens_per_ep_rank=[32620, 32551, 32662, 32904, 32888, 32705, 32928, 32886] avg=32768.0000 max=32928 max_rank=6 min=32551 min_rank=1 imbalance_ratio=1.004883 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=1 tokens_per_ep_rank=[32620, 32551, 32662, 32904, 32888, 32705, 32928, 32886] avg=32768.0000 max=32928 max_rank=6 min=32551 min_rank=1 imbalance_ratio=1.004883 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=1 tokens_per_ep_rank=[32620, 32551, 32662, 32904, 32888, 32705, 32928, 32886] avg=32768.0000 max=32928 max_rank=6 min=32551 min_rank=1 imbalance_ratio=1.004883 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=1 tokens_per_ep_rank=[32620, 32551, 32662, 32904, 32888, 32705, 32928, 32886] avg=32768.0000 max=32928 max_rank=6 min=32551 min_rank=1 imbalance_ratio=1.004883 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=1 tokens_per_ep_rank=[32620, 32551, 32662, 32904, 32888, 32705, 32928, 32886] avg=32768.0000 max=32928 max_rank=6 min=32551 min_rank=1 imbalance_ratio=1.004883 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=1 tokens_per_ep_rank=[32620, 32551, 32662, 32904, 32888, 32705, 32928, 32886] avg=32768.0000 max=32928 max_rank=6 min=32551 min_rank=1 imbalance_ratio=1.004883 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=2 tokens_per_ep_rank=[32721, 32663, 32893, 32675, 32891, 32678, 32933, 32690] avg=32768.0000 max=32933 max_rank=6 min=32663 min_rank=1 imbalance_ratio=1.005035 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=2 tokens_per_ep_rank=[32721, 32663, 32893, 32675, 32891, 32678, 32933, 32690] avg=32768.0000 max=32933 max_rank=6 min=32663 min_rank=1 imbalance_ratio=1.005035 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=2 tokens_per_ep_rank=[32721, 32663, 32893, 32675, 32891, 32678, 32933, 32690] avg=32768.0000 max=32933 max_rank=6 min=32663 min_rank=1 imbalance_ratio=1.005035 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=2 tokens_per_ep_rank=[32721, 32663, 32893, 32675, 32891, 32678, 32933, 32690] avg=32768.0000 max=32933 max_rank=6 min=32663 min_rank=1 imbalance_ratio=1.005035 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=2 tokens_per_ep_rank=[32721, 32663, 32893, 32675, 32891, 32678, 32933, 32690] avg=32768.0000 max=32933 max_rank=6 min=32663 min_rank=1 imbalance_ratio=1.005035 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=1 tokens_per_ep_rank=[32620, 32551, 32662, 32904, 32888, 32705, 32928, 32886] avg=32768.0000 max=32928 max_rank=6 min=32551 min_rank=1 imbalance_ratio=1.004883 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=2 tokens_per_ep_rank=[32721, 32663, 32893, 32675, 32891, 32678, 32933, 32690] avg=32768.0000 max=32933 max_rank=6 min=32663 min_rank=1 imbalance_ratio=1.005035 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=2 tokens_per_ep_rank=[32721, 32663, 32893, 32675, 32891, 32678, 32933, 32690] avg=32768.0000 max=32933 max_rank=6 min=32663 min_rank=1 imbalance_ratio=1.005035 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=2 tokens_per_ep_rank=[32721, 32663, 32893, 32675, 32891, 32678, 32933, 32690] avg=32768.0000 max=32933 max_rank=6 min=32663 min_rank=1 imbalance_ratio=1.005035 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=2 tokens_per_ep_rank=[32721, 32663, 32893, 32675, 32891, 32678, 32933, 32690] avg=32768.0000 max=32933 max_rank=6 min=32663 min_rank=1 imbalance_ratio=1.005035 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=2 tokens_per_ep_rank=[32721, 32663, 32893, 32675, 32891, 32678, 32933, 32690] avg=32768.0000 max=32933 max_rank=6 min=32663 min_rank=1 imbalance_ratio=1.005035 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=2 tokens_per_ep_rank=[32721, 32663, 32893, 32675, 32891, 32678, 32933, 32690] avg=32768.0000 max=32933 max_rank=6 min=32663 min_rank=1 imbalance_ratio=1.005035 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=2 tokens_per_ep_rank=[32721, 32663, 32893, 32675, 32891, 32678, 32933, 32690] avg=32768.0000 max=32933 max_rank=6 min=32663 min_rank=1 imbalance_ratio=1.005035 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=2 tokens_per_ep_rank=[32721, 32663, 32893, 32675, 32891, 32678, 32933, 32690] avg=32768.0000 max=32933 max_rank=6 min=32663 min_rank=1 imbalance_ratio=1.005035 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=2 tokens_per_ep_rank=[32721, 32663, 32893, 32675, 32891, 32678, 32933, 32690] avg=32768.0000 max=32933 max_rank=6 min=32663 min_rank=1 imbalance_ratio=1.005035 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=3 tokens_per_ep_rank=[32980, 32753, 32781, 32896, 32597, 32354, 32790, 32993] avg=32768.0000 max=32993 max_rank=7 min=32354 min_rank=5 imbalance_ratio=1.006866 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=2 tokens_per_ep_rank=[32721, 32663, 32893, 32675, 32891, 32678, 32933, 32690] avg=32768.0000 max=32933 max_rank=6 min=32663 min_rank=1 imbalance_ratio=1.005035 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=3 tokens_per_ep_rank=[32980, 32753, 32781, 32896, 32597, 32354, 32790, 32993] avg=32768.0000 max=32993 max_rank=7 min=32354 min_rank=5 imbalance_ratio=1.006866 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=3 tokens_per_ep_rank=[32980, 32753, 32781, 32896, 32597, 32354, 32790, 32993] avg=32768.0000 max=32993 max_rank=7 min=32354 min_rank=5 imbalance_ratio=1.006866 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=3 tokens_per_ep_rank=[32980, 32753, 32781, 32896, 32597, 32354, 32790, 32993] avg=32768.0000 max=32993 max_rank=7 min=32354 min_rank=5 imbalance_ratio=1.006866 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=3 tokens_per_ep_rank=[32980, 32753, 32781, 32896, 32597, 32354, 32790, 32993] avg=32768.0000 max=32993 max_rank=7 min=32354 min_rank=5 imbalance_ratio=1.006866 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=2 tokens_per_ep_rank=[32721, 32663, 32893, 32675, 32891, 32678, 32933, 32690] avg=32768.0000 max=32933 max_rank=6 min=32663 min_rank=1 imbalance_ratio=1.005035 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=3 tokens_per_ep_rank=[32980, 32753, 32781, 32896, 32597, 32354, 32790, 32993] avg=32768.0000 max=32993 max_rank=7 min=32354 min_rank=5 imbalance_ratio=1.006866 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=3 tokens_per_ep_rank=[32980, 32753, 32781, 32896, 32597, 32354, 32790, 32993] avg=32768.0000 max=32993 max_rank=7 min=32354 min_rank=5 imbalance_ratio=1.006866 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=3 tokens_per_ep_rank=[32980, 32753, 32781, 32896, 32597, 32354, 32790, 32993] avg=32768.0000 max=32993 max_rank=7 min=32354 min_rank=5 imbalance_ratio=1.006866 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=3 tokens_per_ep_rank=[32980, 32753, 32781, 32896, 32597, 32354, 32790, 32993] avg=32768.0000 max=32993 max_rank=7 min=32354 min_rank=5 imbalance_ratio=1.006866 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=3 tokens_per_ep_rank=[32980, 32753, 32781, 32896, 32597, 32354, 32790, 32993] avg=32768.0000 max=32993 max_rank=7 min=32354 min_rank=5 imbalance_ratio=1.006866 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=3 tokens_per_ep_rank=[32980, 32753, 32781, 32896, 32597, 32354, 32790, 32993] avg=32768.0000 max=32993 max_rank=7 min=32354 min_rank=5 imbalance_ratio=1.006866 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=4 tokens_per_ep_rank=[32827, 32620, 33188, 32743, 32603, 32985, 32587, 32591] avg=32768.0000 max=33188 max_rank=2 min=32587 min_rank=6 imbalance_ratio=1.012817 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=3 tokens_per_ep_rank=[32980, 32753, 32781, 32896, 32597, 32354, 32790, 32993] avg=32768.0000 max=32993 max_rank=7 min=32354 min_rank=5 imbalance_ratio=1.006866 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=3 tokens_per_ep_rank=[32980, 32753, 32781, 32896, 32597, 32354, 32790, 32993] avg=32768.0000 max=32993 max_rank=7 min=32354 min_rank=5 imbalance_ratio=1.006866 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=3 tokens_per_ep_rank=[32980, 32753, 32781, 32896, 32597, 32354, 32790, 32993] avg=32768.0000 max=32993 max_rank=7 min=32354 min_rank=5 imbalance_ratio=1.006866 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=4 tokens_per_ep_rank=[32827, 32620, 33188, 32743, 32603, 32985, 32587, 32591] avg=32768.0000 max=33188 max_rank=2 min=32587 min_rank=6 imbalance_ratio=1.012817 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=3 tokens_per_ep_rank=[32980, 32753, 32781, 32896, 32597, 32354, 32790, 32993] avg=32768.0000 max=32993 max_rank=7 min=32354 min_rank=5 imbalance_ratio=1.006866 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=4 tokens_per_ep_rank=[32827, 32620, 33188, 32743, 32603, 32985, 32587, 32591] avg=32768.0000 max=33188 max_rank=2 min=32587 min_rank=6 imbalance_ratio=1.012817 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=4 tokens_per_ep_rank=[32827, 32620, 33188, 32743, 32603, 32985, 32587, 32591] avg=32768.0000 max=33188 max_rank=2 min=32587 min_rank=6 imbalance_ratio=1.012817 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=4 tokens_per_ep_rank=[32827, 32620, 33096, 32743, 32603, 32985, 32679, 32591] avg=32768.0000 max=33096 max_rank=2 min=32591 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=92 max_load_reduction_pct=0.002772 moved_to_replicas=92 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=4 tokens_per_ep_rank=[32827, 32620, 33096, 32743, 32603, 32985, 32679, 32591] avg=32768.0000 max=33096 max_rank=2 min=32591 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=92 max_load_reduction_pct=0.002772 moved_to_replicas=92 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=3 tokens_per_ep_rank=[32980, 32753, 32781, 32896, 32597, 32354, 32790, 32993] avg=32768.0000 max=32993 max_rank=7 min=32354 min_rank=5 imbalance_ratio=1.006866 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=4 tokens_per_ep_rank=[32827, 32620, 33188, 32743, 32603, 32985, 32587, 32591] avg=32768.0000 max=33188 max_rank=2 min=32587 min_rank=6 imbalance_ratio=1.012817 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=4 tokens_per_ep_rank=[32827, 32620, 33188, 32743, 32603, 32985, 32587, 32591] avg=32768.0000 max=33188 max_rank=2 min=32587 min_rank=6 imbalance_ratio=1.012817 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=4 tokens_per_ep_rank=[32827, 32620, 33096, 32743, 32603, 32985, 32679, 32591] avg=32768.0000 max=33096 max_rank=2 min=32591 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=92 max_load_reduction_pct=0.002772 moved_to_replicas=92 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=4 tokens_per_ep_rank=[32827, 32620, 33096, 32743, 32603, 32985, 32679, 32591] avg=32768.0000 max=33096 max_rank=2 min=32591 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=92 max_load_reduction_pct=0.002772 moved_to_replicas=92 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=4 tokens_per_ep_rank=[32827, 32620, 33188, 32743, 32603, 32985, 32587, 32591] avg=32768.0000 max=33188 max_rank=2 min=32587 min_rank=6 imbalance_ratio=1.012817 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=5 tokens_per_ep_rank=[32610, 32906, 32996, 32596, 32525, 32726, 32864, 32921] avg=32768.0000 max=32996 max_rank=2 min=32525 min_rank=4 imbalance_ratio=1.006958 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=4 tokens_per_ep_rank=[32827, 32620, 33096, 32743, 32603, 32985, 32679, 32591] avg=32768.0000 max=33096 max_rank=2 min=32591 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=92 max_load_reduction_pct=0.002772 moved_to_replicas=92 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=4 tokens_per_ep_rank=[32827, 32620, 33096, 32743, 32603, 32985, 32679, 32591] avg=32768.0000 max=33096 max_rank=2 min=32591 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=92 max_load_reduction_pct=0.002772 moved_to_replicas=92 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=5 tokens_per_ep_rank=[32610, 32906, 32996, 32596, 32525, 32726, 32864, 32921] avg=32768.0000 max=32996 max_rank=2 min=32525 min_rank=4 imbalance_ratio=1.006958 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=4 tokens_per_ep_rank=[32827, 32620, 33096, 32743, 32603, 32985, 32679, 32591] avg=32768.0000 max=33096 max_rank=2 min=32591 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=92 max_load_reduction_pct=0.002772 moved_to_replicas=92 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=4 tokens_per_ep_rank=[32827, 32620, 33188, 32743, 32603, 32985, 32587, 32591] avg=32768.0000 max=33188 max_rank=2 min=32587 min_rank=6 imbalance_ratio=1.012817 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=5 tokens_per_ep_rank=[32610, 32906, 32996, 32596, 32525, 32726, 32864, 32921] avg=32768.0000 max=32996 max_rank=2 min=32525 min_rank=4 imbalance_ratio=1.006958 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=5 tokens_per_ep_rank=[32610, 32906, 32996, 32596, 32525, 32726, 32864, 32921] avg=32768.0000 max=32996 max_rank=2 min=32525 min_rank=4 imbalance_ratio=1.006958 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=5 tokens_per_ep_rank=[32610, 32906, 32996, 32596, 32525, 32726, 32864, 32921] avg=32768.0000 max=32996 max_rank=2 min=32525 min_rank=4 imbalance_ratio=1.006958 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=5 tokens_per_ep_rank=[32610, 32906, 32996, 32596, 32525, 32726, 32864, 32921] avg=32768.0000 max=32996 max_rank=2 min=32525 min_rank=4 imbalance_ratio=1.006958 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=5 tokens_per_ep_rank=[32610, 32906, 32996, 32596, 32525, 32726, 32864, 32921] avg=32768.0000 max=32996 max_rank=2 min=32525 min_rank=4 imbalance_ratio=1.006958 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=4 tokens_per_ep_rank=[32827, 32620, 33096, 32743, 32603, 32985, 32679, 32591] avg=32768.0000 max=33096 max_rank=2 min=32591 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=92 max_load_reduction_pct=0.002772 moved_to_replicas=92 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=5 tokens_per_ep_rank=[32610, 32906, 32996, 32596, 32525, 32726, 32864, 32921] avg=32768.0000 max=32996 max_rank=2 min=32525 min_rank=4 imbalance_ratio=1.006958 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=5 tokens_per_ep_rank=[32610, 32906, 32996, 32596, 32525, 32726, 32864, 32921] avg=32768.0000 max=32996 max_rank=2 min=32525 min_rank=4 imbalance_ratio=1.006958 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=5 tokens_per_ep_rank=[32610, 32906, 32996, 32596, 32525, 32726, 32864, 32921] avg=32768.0000 max=32996 max_rank=2 min=32525 min_rank=4 imbalance_ratio=1.006958 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=5 tokens_per_ep_rank=[32610, 32906, 32996, 32596, 32525, 32726, 32864, 32921] avg=32768.0000 max=32996 max_rank=2 min=32525 min_rank=4 imbalance_ratio=1.006958 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=6 tokens_per_ep_rank=[32786, 32971, 32750, 32536, 32869, 32657, 32768, 32807] avg=32768.0000 max=32971 max_rank=1 min=32536 min_rank=3 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=6 tokens_per_ep_rank=[32786, 32971, 32750, 32536, 32869, 32657, 32768, 32807] avg=32768.0000 max=32971 max_rank=1 min=32536 min_rank=3 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=5 tokens_per_ep_rank=[32610, 32906, 32996, 32596, 32525, 32726, 32864, 32921] avg=32768.0000 max=32996 max_rank=2 min=32525 min_rank=4 imbalance_ratio=1.006958 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=5 tokens_per_ep_rank=[32610, 32906, 32996, 32596, 32525, 32726, 32864, 32921] avg=32768.0000 max=32996 max_rank=2 min=32525 min_rank=4 imbalance_ratio=1.006958 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=5 tokens_per_ep_rank=[32610, 32906, 32996, 32596, 32525, 32726, 32864, 32921] avg=32768.0000 max=32996 max_rank=2 min=32525 min_rank=4 imbalance_ratio=1.006958 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=6 tokens_per_ep_rank=[32786, 32971, 32750, 32536, 32869, 32657, 32768, 32807] avg=32768.0000 max=32971 max_rank=1 min=32536 min_rank=3 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=6 tokens_per_ep_rank=[32786, 32971, 32750, 32536, 32869, 32657, 32768, 32807] avg=32768.0000 max=32971 max_rank=1 min=32536 min_rank=3 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=5 tokens_per_ep_rank=[32610, 32906, 32996, 32596, 32525, 32726, 32864, 32921] avg=32768.0000 max=32996 max_rank=2 min=32525 min_rank=4 imbalance_ratio=1.006958 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=6 tokens_per_ep_rank=[32786, 32971, 32750, 32536, 32869, 32657, 32768, 32807] avg=32768.0000 max=32971 max_rank=1 min=32536 min_rank=3 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=6 tokens_per_ep_rank=[32786, 32971, 32750, 32536, 32869, 32657, 32768, 32807] avg=32768.0000 max=32971 max_rank=1 min=32536 min_rank=3 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=6 tokens_per_ep_rank=[32786, 32971, 32750, 32536, 32869, 32657, 32768, 32807] avg=32768.0000 max=32971 max_rank=1 min=32536 min_rank=3 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=6 tokens_per_ep_rank=[32786, 32971, 32750, 32536, 32869, 32657, 32768, 32807] avg=32768.0000 max=32971 max_rank=1 min=32536 min_rank=3 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=5 tokens_per_ep_rank=[32610, 32906, 32996, 32596, 32525, 32726, 32864, 32921] avg=32768.0000 max=32996 max_rank=2 min=32525 min_rank=4 imbalance_ratio=1.006958 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=6 tokens_per_ep_rank=[32786, 32971, 32750, 32536, 32869, 32657, 32768, 32807] avg=32768.0000 max=32971 max_rank=1 min=32536 min_rank=3 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=6 tokens_per_ep_rank=[32786, 32971, 32750, 32536, 32869, 32657, 32768, 32807] avg=32768.0000 max=32971 max_rank=1 min=32536 min_rank=3 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=7 tokens_per_ep_rank=[32665, 32625, 32501, 32804, 32917, 33036, 32826, 32770] avg=32768.0000 max=33036 max_rank=5 min=32501 min_rank=2 imbalance_ratio=1.008179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=6 tokens_per_ep_rank=[32786, 32971, 32750, 32536, 32869, 32657, 32768, 32807] avg=32768.0000 max=32971 max_rank=1 min=32536 min_rank=3 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=7 tokens_per_ep_rank=[32665, 32625, 32501, 32804, 32917, 33036, 32826, 32770] avg=32768.0000 max=33036 max_rank=5 min=32501 min_rank=2 imbalance_ratio=1.008179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=6 tokens_per_ep_rank=[32786, 32971, 32750, 32536, 32869, 32657, 32768, 32807] avg=32768.0000 max=32971 max_rank=1 min=32536 min_rank=3 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=7 tokens_per_ep_rank=[32665, 32625, 32501, 32804, 32917, 33036, 32826, 32770] avg=32768.0000 max=33036 max_rank=5 min=32501 min_rank=2 imbalance_ratio=1.008179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=7 tokens_per_ep_rank=[32665, 32625, 32501, 32804, 32917, 33036, 32826, 32770] avg=32768.0000 max=33036 max_rank=5 min=32501 min_rank=2 imbalance_ratio=1.008179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=6 tokens_per_ep_rank=[32786, 32971, 32750, 32536, 32869, 32657, 32768, 32807] avg=32768.0000 max=32971 max_rank=1 min=32536 min_rank=3 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=6 tokens_per_ep_rank=[32786, 32971, 32750, 32536, 32869, 32657, 32768, 32807] avg=32768.0000 max=32971 max_rank=1 min=32536 min_rank=3 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=7 tokens_per_ep_rank=[32665, 32625, 32501, 32804, 32917, 33036, 32826, 32770] avg=32768.0000 max=33036 max_rank=5 min=32501 min_rank=2 imbalance_ratio=1.008179 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=6 tokens_per_ep_rank=[32786, 32971, 32750, 32536, 32869, 32657, 32768, 32807] avg=32768.0000 max=32971 max_rank=1 min=32536 min_rank=3 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=7 tokens_per_ep_rank=[32665, 32625, 32501, 32804, 32917, 33036, 32826, 32770] avg=32768.0000 max=33036 max_rank=5 min=32501 min_rank=2 imbalance_ratio=1.008179 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=7 tokens_per_ep_rank=[32665, 32625, 32501, 32804, 32917, 33036, 32826, 32770] avg=32768.0000 max=33036 max_rank=5 min=32501 min_rank=2 imbalance_ratio=1.008179 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=7 tokens_per_ep_rank=[32665, 32625, 32501, 32804, 32917, 33036, 32826, 32770] avg=32768.0000 max=33036 max_rank=5 min=32501 min_rank=2 imbalance_ratio=1.008179 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=6 tokens_per_ep_rank=[32786, 32971, 32750, 32536, 32869, 32657, 32768, 32807] avg=32768.0000 max=32971 max_rank=1 min=32536 min_rank=3 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=7 tokens_per_ep_rank=[32665, 32625, 32501, 32804, 32917, 33036, 32826, 32770] avg=32768.0000 max=33036 max_rank=5 min=32501 min_rank=2 imbalance_ratio=1.008179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=7 tokens_per_ep_rank=[32665, 32625, 32501, 32804, 32917, 33036, 32826, 32770] avg=32768.0000 max=33036 max_rank=5 min=32501 min_rank=2 imbalance_ratio=1.008179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=8 tokens_per_ep_rank=[32920, 32501, 32979, 33042, 32684, 32815, 32482, 32721] avg=32768.0000 max=33042 max_rank=3 min=32482 min_rank=6 imbalance_ratio=1.008362 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=7 tokens_per_ep_rank=[32665, 32625, 32501, 32804, 32917, 33036, 32826, 32770] avg=32768.0000 max=33036 max_rank=5 min=32501 min_rank=2 imbalance_ratio=1.008179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=8 tokens_per_ep_rank=[32920, 32501, 32979, 33042, 32684, 32815, 32482, 32721] avg=32768.0000 max=33042 max_rank=3 min=32482 min_rank=6 imbalance_ratio=1.008362 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=7 tokens_per_ep_rank=[32665, 32625, 32501, 32804, 32917, 33036, 32826, 32770] avg=32768.0000 max=33036 max_rank=5 min=32501 min_rank=2 imbalance_ratio=1.008179 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=8 tokens_per_ep_rank=[32920, 32501, 32979, 33042, 32684, 32815, 32482, 32721] avg=32768.0000 max=33042 max_rank=3 min=32482 min_rank=6 imbalance_ratio=1.008362 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=8 tokens_per_ep_rank=[32920, 32501, 32979, 33042, 32684, 32815, 32482, 32721] avg=32768.0000 max=33042 max_rank=3 min=32482 min_rank=6 imbalance_ratio=1.008362 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=7 tokens_per_ep_rank=[32665, 32625, 32501, 32804, 32917, 33036, 32826, 32770] avg=32768.0000 max=33036 max_rank=5 min=32501 min_rank=2 imbalance_ratio=1.008179 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=7 tokens_per_ep_rank=[32665, 32625, 32501, 32804, 32917, 33036, 32826, 32770] avg=32768.0000 max=33036 max_rank=5 min=32501 min_rank=2 imbalance_ratio=1.008179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=8 tokens_per_ep_rank=[32920, 32501, 32979, 33042, 32684, 32815, 32482, 32721] avg=32768.0000 max=33042 max_rank=3 min=32482 min_rank=6 imbalance_ratio=1.008362 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=7 tokens_per_ep_rank=[32665, 32625, 32501, 32804, 32917, 33036, 32826, 32770] avg=32768.0000 max=33036 max_rank=5 min=32501 min_rank=2 imbalance_ratio=1.008179 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=8 tokens_per_ep_rank=[32920, 32501, 32979, 33042, 32684, 32815, 32482, 32721] avg=32768.0000 max=33042 max_rank=3 min=32482 min_rank=6 imbalance_ratio=1.008362 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=8 tokens_per_ep_rank=[32920, 32501, 32979, 33042, 32684, 32815, 32482, 32721] avg=32768.0000 max=33042 max_rank=3 min=32482 min_rank=6 imbalance_ratio=1.008362 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=8 tokens_per_ep_rank=[32920, 32501, 32979, 33042, 32684, 32815, 32482, 32721] avg=32768.0000 max=33042 max_rank=3 min=32482 min_rank=6 imbalance_ratio=1.008362 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=8 tokens_per_ep_rank=[32920, 32501, 32979, 33042, 32684, 32815, 32482, 32721] avg=32768.0000 max=33042 max_rank=3 min=32482 min_rank=6 imbalance_ratio=1.008362 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=7 tokens_per_ep_rank=[32665, 32625, 32501, 32804, 32917, 33036, 32826, 32770] avg=32768.0000 max=33036 max_rank=5 min=32501 min_rank=2 imbalance_ratio=1.008179 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=8 tokens_per_ep_rank=[32920, 32501, 32979, 33042, 32684, 32815, 32482, 32721] avg=32768.0000 max=33042 max_rank=3 min=32482 min_rank=6 imbalance_ratio=1.008362 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=9 tokens_per_ep_rank=[32785, 32530, 32596, 32795, 32895, 33030, 32984, 32529] avg=32768.0000 max=33030 max_rank=5 min=32529 min_rank=7 imbalance_ratio=1.007996 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=9 tokens_per_ep_rank=[32785, 32530, 32596, 32795, 32895, 33030, 32984, 32529] avg=32768.0000 max=33030 max_rank=5 min=32529 min_rank=7 imbalance_ratio=1.007996 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=8 tokens_per_ep_rank=[32920, 32501, 32979, 33042, 32684, 32815, 32482, 32721] avg=32768.0000 max=33042 max_rank=3 min=32482 min_rank=6 imbalance_ratio=1.008362 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=8 tokens_per_ep_rank=[32920, 32501, 32979, 33042, 32684, 32815, 32482, 32721] avg=32768.0000 max=33042 max_rank=3 min=32482 min_rank=6 imbalance_ratio=1.008362 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=9 tokens_per_ep_rank=[32785, 32530, 32596, 32795, 32895, 33030, 32984, 32529] avg=32768.0000 max=33030 max_rank=5 min=32529 min_rank=7 imbalance_ratio=1.007996 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=9 tokens_per_ep_rank=[32785, 32530, 32596, 32795, 32895, 33030, 32984, 32529] avg=32768.0000 max=33030 max_rank=5 min=32529 min_rank=7 imbalance_ratio=1.007996 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=8 tokens_per_ep_rank=[32920, 32501, 32979, 33042, 32684, 32815, 32482, 32721] avg=32768.0000 max=33042 max_rank=3 min=32482 min_rank=6 imbalance_ratio=1.008362 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=8 tokens_per_ep_rank=[32920, 32501, 32979, 33042, 32684, 32815, 32482, 32721] avg=32768.0000 max=33042 max_rank=3 min=32482 min_rank=6 imbalance_ratio=1.008362 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=9 tokens_per_ep_rank=[32785, 32530, 32596, 32795, 32895, 33030, 32984, 32529] avg=32768.0000 max=33030 max_rank=5 min=32529 min_rank=7 imbalance_ratio=1.007996 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=9 tokens_per_ep_rank=[32785, 32530, 32596, 32795, 32895, 33030, 32984, 32529] avg=32768.0000 max=33030 max_rank=5 min=32529 min_rank=7 imbalance_ratio=1.007996 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=8 tokens_per_ep_rank=[32920, 32501, 32979, 33042, 32684, 32815, 32482, 32721] avg=32768.0000 max=33042 max_rank=3 min=32482 min_rank=6 imbalance_ratio=1.008362 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=9 tokens_per_ep_rank=[32785, 32530, 32596, 32795, 32895, 33030, 32984, 32529] avg=32768.0000 max=33030 max_rank=5 min=32529 min_rank=7 imbalance_ratio=1.007996 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=9 tokens_per_ep_rank=[32785, 32530, 32596, 32795, 32895, 33030, 32984, 32529] avg=32768.0000 max=33030 max_rank=5 min=32529 min_rank=7 imbalance_ratio=1.007996 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=9 tokens_per_ep_rank=[32785, 32530, 32596, 32795, 32895, 33030, 32984, 32529] avg=32768.0000 max=33030 max_rank=5 min=32529 min_rank=7 imbalance_ratio=1.007996 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=8 tokens_per_ep_rank=[32920, 32501, 32979, 33042, 32684, 32815, 32482, 32721] avg=32768.0000 max=33042 max_rank=3 min=32482 min_rank=6 imbalance_ratio=1.008362 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=10 tokens_per_ep_rank=[32623, 32965, 32990, 32628, 32572, 32642, 32900, 32824] avg=32768.0000 max=32990 max_rank=2 min=32572 min_rank=4 imbalance_ratio=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=9 tokens_per_ep_rank=[32785, 32530, 32596, 32795, 32895, 33030, 32984, 32529] avg=32768.0000 max=33030 max_rank=5 min=32529 min_rank=7 imbalance_ratio=1.007996 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=10 tokens_per_ep_rank=[32623, 32965, 32990, 32628, 32572, 32642, 32900, 32824] avg=32768.0000 max=32990 max_rank=2 min=32572 min_rank=4 imbalance_ratio=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=9 tokens_per_ep_rank=[32785, 32530, 32596, 32795, 32895, 33030, 32984, 32529] avg=32768.0000 max=33030 max_rank=5 min=32529 min_rank=7 imbalance_ratio=1.007996 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=9 tokens_per_ep_rank=[32785, 32530, 32596, 32795, 32895, 33030, 32984, 32529] avg=32768.0000 max=33030 max_rank=5 min=32529 min_rank=7 imbalance_ratio=1.007996 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=10 tokens_per_ep_rank=[32623, 32965, 32990, 32628, 32572, 32642, 32900, 32824] avg=32768.0000 max=32990 max_rank=2 min=32572 min_rank=4 imbalance_ratio=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=10 tokens_per_ep_rank=[32623, 32965, 32990, 32628, 32572, 32642, 32900, 32824] avg=32768.0000 max=32990 max_rank=2 min=32572 min_rank=4 imbalance_ratio=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=10 tokens_per_ep_rank=[32623, 32965, 32990, 32628, 32572, 32642, 32900, 32824] avg=32768.0000 max=32990 max_rank=2 min=32572 min_rank=4 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=9 tokens_per_ep_rank=[32785, 32530, 32596, 32795, 32895, 33030, 32984, 32529] avg=32768.0000 max=33030 max_rank=5 min=32529 min_rank=7 imbalance_ratio=1.007996 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=9 tokens_per_ep_rank=[32785, 32530, 32596, 32795, 32895, 33030, 32984, 32529] avg=32768.0000 max=33030 max_rank=5 min=32529 min_rank=7 imbalance_ratio=1.007996 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=10 tokens_per_ep_rank=[32623, 32965, 32990, 32628, 32572, 32642, 32900, 32824] avg=32768.0000 max=32990 max_rank=2 min=32572 min_rank=4 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=9 tokens_per_ep_rank=[32785, 32530, 32596, 32795, 32895, 33030, 32984, 32529] avg=32768.0000 max=33030 max_rank=5 min=32529 min_rank=7 imbalance_ratio=1.007996 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=10 tokens_per_ep_rank=[32623, 32965, 32990, 32628, 32572, 32642, 32900, 32824] avg=32768.0000 max=32990 max_rank=2 min=32572 min_rank=4 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=10 tokens_per_ep_rank=[32623, 32965, 32990, 32628, 32572, 32642, 32900, 32824] avg=32768.0000 max=32990 max_rank=2 min=32572 min_rank=4 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=10 tokens_per_ep_rank=[32623, 32965, 32990, 32628, 32572, 32642, 32900, 32824] avg=32768.0000 max=32990 max_rank=2 min=32572 min_rank=4 imbalance_ratio=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=9 tokens_per_ep_rank=[32785, 32530, 32596, 32795, 32895, 33030, 32984, 32529] avg=32768.0000 max=33030 max_rank=5 min=32529 min_rank=7 imbalance_ratio=1.007996 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=10 tokens_per_ep_rank=[32623, 32965, 32990, 32628, 32572, 32642, 32900, 32824] avg=32768.0000 max=32990 max_rank=2 min=32572 min_rank=4 imbalance_ratio=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=10 tokens_per_ep_rank=[32623, 32965, 32990, 32628, 32572, 32642, 32900, 32824] avg=32768.0000 max=32990 max_rank=2 min=32572 min_rank=4 imbalance_ratio=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=10 tokens_per_ep_rank=[32623, 32965, 32990, 32628, 32572, 32642, 32900, 32824] avg=32768.0000 max=32990 max_rank=2 min=32572 min_rank=4 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=10 tokens_per_ep_rank=[32623, 32965, 32990, 32628, 32572, 32642, 32900, 32824] avg=32768.0000 max=32990 max_rank=2 min=32572 min_rank=4 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=10 tokens_per_ep_rank=[32623, 32965, 32990, 32628, 32572, 32642, 32900, 32824] avg=32768.0000 max=32990 max_rank=2 min=32572 min_rank=4 imbalance_ratio=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=10 tokens_per_ep_rank=[32623, 32965, 32990, 32628, 32572, 32642, 32900, 32824] avg=32768.0000 max=32990 max_rank=2 min=32572 min_rank=4 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=10 tokens_per_ep_rank=[32623, 32965, 32990, 32628, 32572, 32642, 32900, 32824] avg=32768.0000 max=32990 max_rank=2 min=32572 min_rank=4 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=7 records=10 total_expected_recv=327628 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76672 peak_demand=33036 required_capacity=1.008179 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=7 records=10 total_expected_recv=327732 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=77184 peak_demand=32993 required_capacity=1.006866 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=7 records=10 total_expected_recv=328244 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76032 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=7 records=10 total_expected_recv=327441 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76416 peak_demand=32917 required_capacity=1.004547 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=7 records=10 total_expected_recv=327085 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=77312 peak_demand=32971 required_capacity=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=7 records=10 total_expected_recv=327619 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76416 peak_demand=33042 required_capacity=1.008362 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=7 records=10 total_expected_recv=328154 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76672 peak_demand=32984 required_capacity=1.006592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=7 records=10 total_expected_recv=327537 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=77056 peak_demand=32980 required_capacity=1.006470 + [2026-08-26 15:53:13] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 213.5 | throughput per GPU (TFLOP/s/GPU): 578.4 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.782492E+00 | loss scale: 1.0 | grad norm: 5.408 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 33664 | min safe capacity: 1.0273 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=0 calls=1 budget=40960 expected_recv=32226 expected_overflow=0 exact_recv_per_rank=[32736, 33078, 32922, 32913, 32573, 32226, 32861, 32835] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=0 calls=1 budget=40960 expected_recv=32922 expected_overflow=0 exact_recv_per_rank=[32736, 33078, 32922, 32913, 32573, 32226, 32861, 32835] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=0 calls=1 budget=40960 expected_recv=32835 expected_overflow=0 exact_recv_per_rank=[32736, 33078, 32922, 32913, 32573, 32226, 32861, 32835] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=0 calls=1 budget=40960 expected_recv=32573 expected_overflow=0 exact_recv_per_rank=[32736, 33078, 32922, 32913, 32573, 32226, 32861, 32835] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=0 calls=1 budget=40960 expected_recv=32861 expected_overflow=0 exact_recv_per_rank=[32736, 33078, 32922, 32913, 32573, 32226, 32861, 32835] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=0 calls=1 budget=40960 expected_recv=33078 expected_overflow=0 exact_recv_per_rank=[32736, 33078, 32922, 32913, 32573, 32226, 32861, 32835] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=0 calls=1 budget=40960 expected_recv=32913 expected_overflow=0 exact_recv_per_rank=[32736, 33078, 32922, 32913, 32573, 32226, 32861, 32835] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=0 calls=1 budget=40960 expected_recv=32226 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=0 calls=1 budget=40960 expected_recv=32922 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=0 calls=1 budget=40960 expected_recv=32835 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=0 calls=1 budget=40960 expected_recv=32573 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=0 calls=1 budget=40960 expected_recv=32736 expected_overflow=0 exact_recv_per_rank=[32736, 33078, 32922, 32913, 32573, 32226, 32861, 32835] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=0 calls=1 budget=40960 expected_recv=32861 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=0 calls=1 budget=40960 expected_recv=33078 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=0 calls=1 budget=40960 expected_recv=32913 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=0 calls=1 budget=40960 expected_recv=32736 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=1 calls=1 budget=40960 expected_recv=32619 expected_overflow=0 exact_recv_per_rank=[32845, 32725, 32833, 32640, 32608, 32619, 32778, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32845, 32725, 32833, 32640, 32608, 32619, 32778, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=1 calls=1 budget=40960 expected_recv=32833 expected_overflow=0 exact_recv_per_rank=[32845, 32725, 32833, 32640, 32608, 32619, 32778, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=1 calls=1 budget=40960 expected_recv=32608 expected_overflow=0 exact_recv_per_rank=[32845, 32725, 32833, 32640, 32608, 32619, 32778, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=1 calls=1 budget=40960 expected_recv=32778 expected_overflow=0 exact_recv_per_rank=[32845, 32725, 32833, 32640, 32608, 32619, 32778, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=1 calls=1 budget=40960 expected_recv=32619 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=1 calls=1 budget=40960 expected_recv=32725 expected_overflow=0 exact_recv_per_rank=[32845, 32725, 32833, 32640, 32608, 32619, 32778, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=1 calls=1 budget=40960 expected_recv=32833 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=1 calls=1 budget=40960 expected_recv=32608 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=1 calls=1 budget=40960 expected_recv=32778 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=1 calls=1 budget=40960 expected_recv=32640 expected_overflow=0 exact_recv_per_rank=[32845, 32725, 32833, 32640, 32608, 32619, 32778, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=1 calls=1 budget=40960 expected_recv=32725 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=1 calls=1 budget=40960 expected_recv=32845 expected_overflow=0 exact_recv_per_rank=[32845, 32725, 32833, 32640, 32608, 32619, 32778, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=1 calls=1 budget=40960 expected_recv=32640 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=1 calls=1 budget=40960 expected_recv=32845 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=2 calls=1 budget=40960 expected_recv=32983 expected_overflow=0 exact_recv_per_rank=[32685, 32883, 32615, 32752, 32562, 32983, 32963, 32701] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=2 calls=1 budget=40960 expected_recv=32701 expected_overflow=0 exact_recv_per_rank=[32685, 32883, 32615, 32752, 32562, 32983, 32963, 32701] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=2 calls=1 budget=40960 expected_recv=32615 expected_overflow=0 exact_recv_per_rank=[32685, 32883, 32615, 32752, 32562, 32983, 32963, 32701] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=2 calls=1 budget=40960 expected_recv=32963 expected_overflow=0 exact_recv_per_rank=[32685, 32883, 32615, 32752, 32562, 32983, 32963, 32701] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=2 calls=1 budget=40960 expected_recv=32562 expected_overflow=0 exact_recv_per_rank=[32685, 32883, 32615, 32752, 32562, 32983, 32963, 32701] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=2 calls=1 budget=40960 expected_recv=32983 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=2 calls=1 budget=40960 expected_recv=32701 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=2 calls=1 budget=40960 expected_recv=32615 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=2 calls=1 budget=40960 expected_recv=32883 expected_overflow=0 exact_recv_per_rank=[32685, 32883, 32615, 32752, 32562, 32983, 32963, 32701] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=2 calls=1 budget=40960 expected_recv=32963 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=2 calls=1 budget=40960 expected_recv=32562 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=2 calls=1 budget=40960 expected_recv=32752 expected_overflow=0 exact_recv_per_rank=[32685, 32883, 32615, 32752, 32562, 32983, 32963, 32701] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=2 calls=1 budget=40960 expected_recv=32883 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=2 calls=1 budget=40960 expected_recv=32685 expected_overflow=0 exact_recv_per_rank=[32685, 32883, 32615, 32752, 32562, 32983, 32963, 32701] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=2 calls=1 budget=40960 expected_recv=32752 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=2 calls=1 budget=40960 expected_recv=32685 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=3 calls=1 budget=40960 expected_recv=32710 expected_overflow=0 exact_recv_per_rank=[32704, 32535, 32850, 32985, 32655, 32710, 32866, 32839] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=3 calls=1 budget=40960 expected_recv=32839 expected_overflow=0 exact_recv_per_rank=[32704, 32535, 32850, 32985, 32655, 32710, 32866, 32839] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=3 calls=1 budget=40960 expected_recv=32850 expected_overflow=0 exact_recv_per_rank=[32704, 32535, 32850, 32985, 32655, 32710, 32866, 32839] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=3 calls=1 budget=40960 expected_recv=32710 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=3 calls=1 budget=40960 expected_recv=32839 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=3 calls=1 budget=40960 expected_recv=32866 expected_overflow=0 exact_recv_per_rank=[32704, 32535, 32850, 32985, 32655, 32710, 32866, 32839] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=3 calls=1 budget=40960 expected_recv=32850 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=3 calls=1 budget=40960 expected_recv=32655 expected_overflow=0 exact_recv_per_rank=[32704, 32535, 32850, 32985, 32655, 32710, 32866, 32839] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=3 calls=1 budget=40960 expected_recv=32535 expected_overflow=0 exact_recv_per_rank=[32704, 32535, 32850, 32985, 32655, 32710, 32866, 32839] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=3 calls=1 budget=40960 expected_recv=32866 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=3 calls=1 budget=40960 expected_recv=32985 expected_overflow=0 exact_recv_per_rank=[32704, 32535, 32850, 32985, 32655, 32710, 32866, 32839] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=3 calls=1 budget=40960 expected_recv=32655 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=3 calls=1 budget=40960 expected_recv=32535 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=3 calls=1 budget=40960 expected_recv=32704 expected_overflow=0 exact_recv_per_rank=[32704, 32535, 32850, 32985, 32655, 32710, 32866, 32839] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=3 calls=1 budget=40960 expected_recv=32985 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=3 calls=1 budget=40960 expected_recv=32704 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=4 calls=1 budget=40960 expected_recv=32963 expected_overflow=0 exact_recv_per_rank=[32574, 32649, 32886, 32841, 32742, 32963, 32697, 32792] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=4 calls=1 budget=40960 expected_recv=32792 expected_overflow=0 exact_recv_per_rank=[32574, 32649, 32886, 32841, 32742, 32963, 32697, 32792] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=4 calls=1 budget=40960 expected_recv=32963 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=4 calls=1 budget=40960 expected_recv=32886 expected_overflow=0 exact_recv_per_rank=[32574, 32649, 32886, 32841, 32742, 32963, 32697, 32792] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=4 calls=1 budget=40960 expected_recv=32792 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=4 calls=1 budget=40960 expected_recv=32886 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=4 calls=1 budget=40960 expected_recv=32697 expected_overflow=0 exact_recv_per_rank=[32574, 32649, 32886, 32841, 32742, 32963, 32697, 32792] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=4 calls=1 budget=40960 expected_recv=32742 expected_overflow=0 exact_recv_per_rank=[32574, 32649, 32886, 32841, 32742, 32963, 32697, 32792] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=4 calls=1 budget=40960 expected_recv=32649 expected_overflow=0 exact_recv_per_rank=[32574, 32649, 32886, 32841, 32742, 32963, 32697, 32792] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=4 calls=1 budget=40960 expected_recv=32697 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=4 calls=1 budget=40960 expected_recv=32841 expected_overflow=0 exact_recv_per_rank=[32574, 32649, 32886, 32841, 32742, 32963, 32697, 32792] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=4 calls=1 budget=40960 expected_recv=32742 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=4 calls=1 budget=40960 expected_recv=32574 expected_overflow=0 exact_recv_per_rank=[32574, 32649, 32886, 32841, 32742, 32963, 32697, 32792] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=4 calls=1 budget=40960 expected_recv=32649 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=4 calls=1 budget=40960 expected_recv=32841 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=4 calls=1 budget=40960 expected_recv=32574 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=5 calls=1 budget=40960 expected_recv=32648 expected_overflow=0 exact_recv_per_rank=[33034, 32644, 33045, 32642, 32569, 32648, 32783, 32779] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=5 calls=1 budget=40960 expected_recv=32779 expected_overflow=0 exact_recv_per_rank=[33034, 32644, 33045, 32642, 32569, 32648, 32783, 32779] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=5 calls=1 budget=40960 expected_recv=32648 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=5 calls=1 budget=40960 expected_recv=33045 expected_overflow=0 exact_recv_per_rank=[33034, 32644, 33045, 32642, 32569, 32648, 32783, 32779] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=5 calls=1 budget=40960 expected_recv=32779 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=5 calls=1 budget=40960 expected_recv=33045 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=5 calls=1 budget=40960 expected_recv=32783 expected_overflow=0 exact_recv_per_rank=[33034, 32644, 33045, 32642, 32569, 32648, 32783, 32779] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=5 calls=1 budget=40960 expected_recv=32569 expected_overflow=0 exact_recv_per_rank=[33034, 32644, 33045, 32642, 32569, 32648, 32783, 32779] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=5 calls=1 budget=40960 expected_recv=32644 expected_overflow=0 exact_recv_per_rank=[33034, 32644, 33045, 32642, 32569, 32648, 32783, 32779] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=5 calls=1 budget=40960 expected_recv=32783 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=5 calls=1 budget=40960 expected_recv=32569 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=5 calls=1 budget=40960 expected_recv=33034 expected_overflow=0 exact_recv_per_rank=[33034, 32644, 33045, 32642, 32569, 32648, 32783, 32779] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=5 calls=1 budget=40960 expected_recv=32644 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=5 calls=1 budget=40960 expected_recv=32642 expected_overflow=0 exact_recv_per_rank=[33034, 32644, 33045, 32642, 32569, 32648, 32783, 32779] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=6 calls=1 budget=40960 expected_recv=32709 expected_overflow=0 exact_recv_per_rank=[32659, 32988, 32788, 32902, 32609, 32709, 32813, 32676] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=5 calls=1 budget=40960 expected_recv=33034 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=5 calls=1 budget=40960 expected_recv=32642 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=6 calls=1 budget=40960 expected_recv=32676 expected_overflow=0 exact_recv_per_rank=[32659, 32988, 32788, 32902, 32609, 32709, 32813, 32676] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=6 calls=1 budget=40960 expected_recv=32709 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=6 calls=1 budget=40960 expected_recv=32788 expected_overflow=0 exact_recv_per_rank=[32659, 32988, 32788, 32902, 32609, 32709, 32813, 32676] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=6 calls=1 budget=40960 expected_recv=32676 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=6 calls=1 budget=40960 expected_recv=32788 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=6 calls=1 budget=40960 expected_recv=32813 expected_overflow=0 exact_recv_per_rank=[32659, 32988, 32788, 32902, 32609, 32709, 32813, 32676] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=6 calls=1 budget=40960 expected_recv=32609 expected_overflow=0 exact_recv_per_rank=[32659, 32988, 32788, 32902, 32609, 32709, 32813, 32676] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=6 calls=1 budget=40960 expected_recv=32813 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=6 calls=1 budget=40960 expected_recv=32988 expected_overflow=0 exact_recv_per_rank=[32659, 32988, 32788, 32902, 32609, 32709, 32813, 32676] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=6 calls=1 budget=40960 expected_recv=32609 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=6 calls=1 budget=40960 expected_recv=32988 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=6 calls=1 budget=40960 expected_recv=32659 expected_overflow=0 exact_recv_per_rank=[32659, 32988, 32788, 32902, 32609, 32709, 32813, 32676] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=6 calls=1 budget=40960 expected_recv=32902 expected_overflow=0 exact_recv_per_rank=[32659, 32988, 32788, 32902, 32609, 32709, 32813, 32676] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=7 calls=1 budget=40960 expected_recv=32722 expected_overflow=0 exact_recv_per_rank=[32431, 32772, 32726, 32651, 32898, 32722, 32916, 33028] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=7 calls=1 budget=40960 expected_recv=33028 expected_overflow=0 exact_recv_per_rank=[32431, 32772, 32726, 32651, 32898, 32722, 32916, 33028] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=6 calls=1 budget=40960 expected_recv=32659 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=7 calls=1 budget=40960 expected_recv=32722 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=6 calls=1 budget=40960 expected_recv=32902 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=7 calls=1 budget=40960 expected_recv=32726 expected_overflow=0 exact_recv_per_rank=[32431, 32772, 32726, 32651, 32898, 32722, 32916, 33028] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=7 calls=1 budget=40960 expected_recv=33028 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=7 calls=1 budget=40960 expected_recv=32726 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=7 calls=1 budget=40960 expected_recv=32916 expected_overflow=0 exact_recv_per_rank=[32431, 32772, 32726, 32651, 32898, 32722, 32916, 33028] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=7 calls=1 budget=40960 expected_recv=32916 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=7 calls=1 budget=40960 expected_recv=32898 expected_overflow=0 exact_recv_per_rank=[32431, 32772, 32726, 32651, 32898, 32722, 32916, 33028] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=7 calls=1 budget=40960 expected_recv=32772 expected_overflow=0 exact_recv_per_rank=[32431, 32772, 32726, 32651, 32898, 32722, 32916, 33028] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=7 calls=1 budget=40960 expected_recv=32898 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=7 calls=1 budget=40960 expected_recv=32772 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=7 calls=1 budget=40960 expected_recv=32431 expected_overflow=0 exact_recv_per_rank=[32431, 32772, 32726, 32651, 32898, 32722, 32916, 33028] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=8 calls=1 budget=40960 expected_recv=32504 expected_overflow=0 exact_recv_per_rank=[33067, 32796, 32871, 32745, 32745, 32504, 32612, 32804] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=7 calls=1 budget=40960 expected_recv=32651 expected_overflow=0 exact_recv_per_rank=[32431, 32772, 32726, 32651, 32898, 32722, 32916, 33028] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=8 calls=1 budget=40960 expected_recv=32804 expected_overflow=0 exact_recv_per_rank=[33067, 32796, 32871, 32745, 32745, 32504, 32612, 32804] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=7 calls=1 budget=40960 expected_recv=32431 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=8 calls=1 budget=40960 expected_recv=32504 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=7 calls=1 budget=40960 expected_recv=32651 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=8 calls=1 budget=40960 expected_recv=32804 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=8 calls=1 budget=40960 expected_recv=32871 expected_overflow=0 exact_recv_per_rank=[33067, 32796, 32871, 32745, 32745, 32504, 32612, 32804] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=8 calls=1 budget=40960 expected_recv=32612 expected_overflow=0 exact_recv_per_rank=[33067, 32796, 32871, 32745, 32745, 32504, 32612, 32804] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=8 calls=1 budget=40960 expected_recv=32871 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=8 calls=1 budget=40960 expected_recv=32745 expected_overflow=0 exact_recv_per_rank=[33067, 32796, 32871, 32745, 32745, 32504, 32612, 32804] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=8 calls=1 budget=40960 expected_recv=32612 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=8 calls=1 budget=40960 expected_recv=32796 expected_overflow=0 exact_recv_per_rank=[33067, 32796, 32871, 32745, 32745, 32504, 32612, 32804] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=8 calls=1 budget=40960 expected_recv=32745 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=8 calls=1 budget=40960 expected_recv=32796 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=8 calls=1 budget=40960 expected_recv=33067 expected_overflow=0 exact_recv_per_rank=[33067, 32796, 32871, 32745, 32745, 32504, 32612, 32804] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=9 calls=1 budget=40960 expected_recv=32562 expected_overflow=0 exact_recv_per_rank=[33051, 32723, 32667, 32642, 32866, 32562, 33096, 32537] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=9 calls=1 budget=40960 expected_recv=32537 expected_overflow=0 exact_recv_per_rank=[33051, 32723, 32667, 32642, 32866, 32562, 33096, 32537] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=8 calls=1 budget=40960 expected_recv=32745 expected_overflow=0 exact_recv_per_rank=[33067, 32796, 32871, 32745, 32745, 32504, 32612, 32804] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=8 calls=1 budget=40960 expected_recv=33067 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=9 calls=1 budget=40960 expected_recv=32562 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=9 calls=1 budget=40960 expected_recv=32667 expected_overflow=0 exact_recv_per_rank=[33051, 32723, 32667, 32642, 32866, 32562, 33096, 32537] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=8 calls=1 budget=40960 expected_recv=32745 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=9 calls=1 budget=40960 expected_recv=32537 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=9 calls=1 budget=40960 expected_recv=32667 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33051, 32723, 32667, 32642, 32866, 32562, 33096, 32537] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=9 calls=1 budget=40960 expected_recv=32866 expected_overflow=0 exact_recv_per_rank=[33051, 32723, 32667, 32642, 32866, 32562, 33096, 32537] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=9 calls=1 budget=40960 expected_recv=32723 expected_overflow=0 exact_recv_per_rank=[33051, 32723, 32667, 32642, 32866, 32562, 33096, 32537] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=9 calls=1 budget=40960 expected_recv=32866 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=9 calls=1 budget=40960 expected_recv=32723 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=9 calls=1 budget=40960 expected_recv=33051 expected_overflow=0 exact_recv_per_rank=[33051, 32723, 32667, 32642, 32866, 32562, 33096, 32537] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=1 tokens_per_ep_rank=[32736, 33078, 32922, 32913, 32573, 32226, 32861, 32835] avg=32768.0000 max=33078 max_rank=1 min=32226 min_rank=5 imbalance_ratio=1.009460 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=9 calls=1 budget=40960 expected_recv=33051 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=9 calls=1 budget=40960 expected_recv=32642 expected_overflow=0 exact_recv_per_rank=[33051, 32723, 32667, 32642, 32866, 32562, 33096, 32537] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=1 tokens_per_ep_rank=[32736, 33078, 32922, 32913, 32573, 32226, 32861, 32835] avg=32768.0000 max=33078 max_rank=1 min=32226 min_rank=5 imbalance_ratio=1.009460 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=1 tokens_per_ep_rank=[32736, 33078, 32922, 32913, 32573, 32226, 32861, 32835] avg=32768.0000 max=33078 max_rank=1 min=32226 min_rank=5 imbalance_ratio=1.009460 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=9 calls=1 budget=40960 expected_recv=32642 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=1 tokens_per_ep_rank=[32736, 33078, 32922, 32913, 32573, 32226, 32861, 32835] avg=32768.0000 max=33078 max_rank=1 min=32226 min_rank=5 imbalance_ratio=1.009460 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=1 tokens_per_ep_rank=[32736, 33078, 32922, 32913, 32573, 32226, 32861, 32835] avg=32768.0000 max=33078 max_rank=1 min=32226 min_rank=5 imbalance_ratio=1.009460 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=1 tokens_per_ep_rank=[32736, 33078, 32922, 32913, 32573, 32226, 32861, 32835] avg=32768.0000 max=33078 max_rank=1 min=32226 min_rank=5 imbalance_ratio=1.009460 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=1 tokens_per_ep_rank=[32736, 33078, 32922, 32913, 32573, 32226, 32861, 32835] avg=32768.0000 max=33078 max_rank=1 min=32226 min_rank=5 imbalance_ratio=1.009460 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=1 tokens_per_ep_rank=[32736, 33078, 32922, 32913, 32573, 32226, 32861, 32835] avg=32768.0000 max=33078 max_rank=1 min=32226 min_rank=5 imbalance_ratio=1.009460 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=1 tokens_per_ep_rank=[32736, 33078, 32922, 32913, 32573, 32226, 32861, 32835] avg=32768.0000 max=33078 max_rank=1 min=32226 min_rank=5 imbalance_ratio=1.009460 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=1 tokens_per_ep_rank=[32736, 33078, 32922, 32913, 32573, 32226, 32861, 32835] avg=32768.0000 max=33078 max_rank=1 min=32226 min_rank=5 imbalance_ratio=1.009460 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=1 tokens_per_ep_rank=[32736, 33078, 32922, 32913, 32573, 32226, 32861, 32835] avg=32768.0000 max=33078 max_rank=1 min=32226 min_rank=5 imbalance_ratio=1.009460 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=1 tokens_per_ep_rank=[32736, 33078, 32922, 32913, 32573, 32226, 32861, 32835] avg=32768.0000 max=33078 max_rank=1 min=32226 min_rank=5 imbalance_ratio=1.009460 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=1 tokens_per_ep_rank=[32736, 33078, 32922, 32913, 32573, 32226, 32861, 32835] avg=32768.0000 max=33078 max_rank=1 min=32226 min_rank=5 imbalance_ratio=1.009460 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=1 tokens_per_ep_rank=[32736, 33078, 32922, 32913, 32573, 32226, 32861, 32835] avg=32768.0000 max=33078 max_rank=1 min=32226 min_rank=5 imbalance_ratio=1.009460 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=2 tokens_per_ep_rank=[32845, 32725, 32833, 32561, 32608, 32619, 32778, 33175] avg=32768.0000 max=33175 max_rank=7 min=32561 min_rank=3 imbalance_ratio=1.012421 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=2 tokens_per_ep_rank=[32845, 32725, 32833, 32561, 32608, 32619, 32778, 33175] avg=32768.0000 max=33175 max_rank=7 min=32561 min_rank=3 imbalance_ratio=1.012421 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=2 tokens_per_ep_rank=[32845, 32725, 32833, 32561, 32608, 32619, 32778, 33175] avg=32768.0000 max=33175 max_rank=7 min=32561 min_rank=3 imbalance_ratio=1.012421 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=1 tokens_per_ep_rank=[32736, 33078, 32922, 32913, 32573, 32226, 32861, 32835] avg=32768.0000 max=33078 max_rank=1 min=32226 min_rank=5 imbalance_ratio=1.009460 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=2 tokens_per_ep_rank=[32845, 32725, 32833, 32561, 32608, 32619, 32778, 33175] avg=32768.0000 max=33175 max_rank=7 min=32561 min_rank=3 imbalance_ratio=1.012421 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=2 tokens_per_ep_rank=[32845, 32725, 32833, 32640, 32608, 32619, 32778, 33096] avg=32768.0000 max=33096 max_rank=7 min=32608 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=79 max_load_reduction_pct=0.002381 moved_to_replicas=79 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=1 tokens_per_ep_rank=[32736, 33078, 32922, 32913, 32573, 32226, 32861, 32835] avg=32768.0000 max=33078 max_rank=1 min=32226 min_rank=5 imbalance_ratio=1.009460 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=2 tokens_per_ep_rank=[32845, 32725, 32833, 32640, 32608, 32619, 32778, 33096] avg=32768.0000 max=33096 max_rank=7 min=32608 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=79 max_load_reduction_pct=0.002381 moved_to_replicas=79 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=2 tokens_per_ep_rank=[32845, 32725, 32833, 32561, 32608, 32619, 32778, 33175] avg=32768.0000 max=33175 max_rank=7 min=32561 min_rank=3 imbalance_ratio=1.012421 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=2 tokens_per_ep_rank=[32845, 32725, 32833, 32640, 32608, 32619, 32778, 33096] avg=32768.0000 max=33096 max_rank=7 min=32608 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=79 max_load_reduction_pct=0.002381 moved_to_replicas=79 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=2 tokens_per_ep_rank=[32845, 32725, 32833, 32561, 32608, 32619, 32778, 33175] avg=32768.0000 max=33175 max_rank=7 min=32561 min_rank=3 imbalance_ratio=1.012421 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=2 tokens_per_ep_rank=[32845, 32725, 32833, 32640, 32608, 32619, 32778, 33096] avg=32768.0000 max=33096 max_rank=7 min=32608 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=79 max_load_reduction_pct=0.002381 moved_to_replicas=79 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=2 tokens_per_ep_rank=[32845, 32725, 32833, 32561, 32608, 32619, 32778, 33175] avg=32768.0000 max=33175 max_rank=7 min=32561 min_rank=3 imbalance_ratio=1.012421 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=2 tokens_per_ep_rank=[32845, 32725, 32833, 32640, 32608, 32619, 32778, 33096] avg=32768.0000 max=33096 max_rank=7 min=32608 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=79 max_load_reduction_pct=0.002381 moved_to_replicas=79 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=3 tokens_per_ep_rank=[32685, 32883, 32615, 32752, 32562, 32983, 32963, 32701] avg=32768.0000 max=32983 max_rank=5 min=32562 min_rank=4 imbalance_ratio=1.006561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=2 tokens_per_ep_rank=[32845, 32725, 32833, 32640, 32608, 32619, 32778, 33096] avg=32768.0000 max=33096 max_rank=7 min=32608 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=79 max_load_reduction_pct=0.002381 moved_to_replicas=79 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=3 tokens_per_ep_rank=[32685, 32883, 32615, 32752, 32562, 32983, 32963, 32701] avg=32768.0000 max=32983 max_rank=5 min=32562 min_rank=4 imbalance_ratio=1.006561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=2 tokens_per_ep_rank=[32845, 32725, 32833, 32561, 32608, 32619, 32778, 33175] avg=32768.0000 max=33175 max_rank=7 min=32561 min_rank=3 imbalance_ratio=1.012421 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=3 tokens_per_ep_rank=[32685, 32883, 32615, 32752, 32562, 32983, 32963, 32701] avg=32768.0000 max=32983 max_rank=5 min=32562 min_rank=4 imbalance_ratio=1.006561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=2 tokens_per_ep_rank=[32845, 32725, 32833, 32640, 32608, 32619, 32778, 33096] avg=32768.0000 max=33096 max_rank=7 min=32608 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=79 max_load_reduction_pct=0.002381 moved_to_replicas=79 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=3 tokens_per_ep_rank=[32685, 32883, 32615, 32752, 32562, 32983, 32963, 32701] avg=32768.0000 max=32983 max_rank=5 min=32562 min_rank=4 imbalance_ratio=1.006561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=3 tokens_per_ep_rank=[32685, 32883, 32615, 32752, 32562, 32983, 32963, 32701] avg=32768.0000 max=32983 max_rank=5 min=32562 min_rank=4 imbalance_ratio=1.006561 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=3 tokens_per_ep_rank=[32685, 32883, 32615, 32752, 32562, 32983, 32963, 32701] avg=32768.0000 max=32983 max_rank=5 min=32562 min_rank=4 imbalance_ratio=1.006561 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=2 tokens_per_ep_rank=[32845, 32725, 32833, 32640, 32608, 32619, 32778, 33096] avg=32768.0000 max=33096 max_rank=7 min=32608 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=79 max_load_reduction_pct=0.002381 moved_to_replicas=79 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=3 tokens_per_ep_rank=[32685, 32883, 32615, 32752, 32562, 32983, 32963, 32701] avg=32768.0000 max=32983 max_rank=5 min=32562 min_rank=4 imbalance_ratio=1.006561 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=3 tokens_per_ep_rank=[32685, 32883, 32615, 32752, 32562, 32983, 32963, 32701] avg=32768.0000 max=32983 max_rank=5 min=32562 min_rank=4 imbalance_ratio=1.006561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=3 tokens_per_ep_rank=[32685, 32883, 32615, 32752, 32562, 32983, 32963, 32701] avg=32768.0000 max=32983 max_rank=5 min=32562 min_rank=4 imbalance_ratio=1.006561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=3 tokens_per_ep_rank=[32685, 32883, 32615, 32752, 32562, 32983, 32963, 32701] avg=32768.0000 max=32983 max_rank=5 min=32562 min_rank=4 imbalance_ratio=1.006561 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=3 tokens_per_ep_rank=[32685, 32883, 32615, 32752, 32562, 32983, 32963, 32701] avg=32768.0000 max=32983 max_rank=5 min=32562 min_rank=4 imbalance_ratio=1.006561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=4 tokens_per_ep_rank=[32704, 32535, 32850, 32985, 32655, 32710, 32866, 32839] avg=32768.0000 max=32985 max_rank=3 min=32535 min_rank=1 imbalance_ratio=1.006622 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=3 tokens_per_ep_rank=[32685, 32883, 32615, 32752, 32562, 32983, 32963, 32701] avg=32768.0000 max=32983 max_rank=5 min=32562 min_rank=4 imbalance_ratio=1.006561 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=4 tokens_per_ep_rank=[32704, 32535, 32850, 32985, 32655, 32710, 32866, 32839] avg=32768.0000 max=32985 max_rank=3 min=32535 min_rank=1 imbalance_ratio=1.006622 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=3 tokens_per_ep_rank=[32685, 32883, 32615, 32752, 32562, 32983, 32963, 32701] avg=32768.0000 max=32983 max_rank=5 min=32562 min_rank=4 imbalance_ratio=1.006561 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=3 tokens_per_ep_rank=[32685, 32883, 32615, 32752, 32562, 32983, 32963, 32701] avg=32768.0000 max=32983 max_rank=5 min=32562 min_rank=4 imbalance_ratio=1.006561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=4 tokens_per_ep_rank=[32704, 32535, 32850, 32985, 32655, 32710, 32866, 32839] avg=32768.0000 max=32985 max_rank=3 min=32535 min_rank=1 imbalance_ratio=1.006622 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=3 tokens_per_ep_rank=[32685, 32883, 32615, 32752, 32562, 32983, 32963, 32701] avg=32768.0000 max=32983 max_rank=5 min=32562 min_rank=4 imbalance_ratio=1.006561 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=4 tokens_per_ep_rank=[32704, 32535, 32850, 32985, 32655, 32710, 32866, 32839] avg=32768.0000 max=32985 max_rank=3 min=32535 min_rank=1 imbalance_ratio=1.006622 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=4 tokens_per_ep_rank=[32704, 32535, 32850, 32985, 32655, 32710, 32866, 32839] avg=32768.0000 max=32985 max_rank=3 min=32535 min_rank=1 imbalance_ratio=1.006622 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=4 tokens_per_ep_rank=[32704, 32535, 32850, 32985, 32655, 32710, 32866, 32839] avg=32768.0000 max=32985 max_rank=3 min=32535 min_rank=1 imbalance_ratio=1.006622 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=4 tokens_per_ep_rank=[32704, 32535, 32850, 32985, 32655, 32710, 32866, 32839] avg=32768.0000 max=32985 max_rank=3 min=32535 min_rank=1 imbalance_ratio=1.006622 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=4 tokens_per_ep_rank=[32704, 32535, 32850, 32985, 32655, 32710, 32866, 32839] avg=32768.0000 max=32985 max_rank=3 min=32535 min_rank=1 imbalance_ratio=1.006622 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=4 tokens_per_ep_rank=[32704, 32535, 32850, 32985, 32655, 32710, 32866, 32839] avg=32768.0000 max=32985 max_rank=3 min=32535 min_rank=1 imbalance_ratio=1.006622 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=3 tokens_per_ep_rank=[32685, 32883, 32615, 32752, 32562, 32983, 32963, 32701] avg=32768.0000 max=32983 max_rank=5 min=32562 min_rank=4 imbalance_ratio=1.006561 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=4 tokens_per_ep_rank=[32704, 32535, 32850, 32985, 32655, 32710, 32866, 32839] avg=32768.0000 max=32985 max_rank=3 min=32535 min_rank=1 imbalance_ratio=1.006622 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=5 tokens_per_ep_rank=[32574, 32649, 32886, 32841, 32742, 32963, 32697, 32792] avg=32768.0000 max=32963 max_rank=5 min=32574 min_rank=0 imbalance_ratio=1.005951 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=4 tokens_per_ep_rank=[32704, 32535, 32850, 32985, 32655, 32710, 32866, 32839] avg=32768.0000 max=32985 max_rank=3 min=32535 min_rank=1 imbalance_ratio=1.006622 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=4 tokens_per_ep_rank=[32704, 32535, 32850, 32985, 32655, 32710, 32866, 32839] avg=32768.0000 max=32985 max_rank=3 min=32535 min_rank=1 imbalance_ratio=1.006622 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=5 tokens_per_ep_rank=[32574, 32649, 32886, 32841, 32742, 32963, 32697, 32792] avg=32768.0000 max=32963 max_rank=5 min=32574 min_rank=0 imbalance_ratio=1.005951 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=4 tokens_per_ep_rank=[32704, 32535, 32850, 32985, 32655, 32710, 32866, 32839] avg=32768.0000 max=32985 max_rank=3 min=32535 min_rank=1 imbalance_ratio=1.006622 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=5 tokens_per_ep_rank=[32574, 32649, 32886, 32841, 32742, 32963, 32697, 32792] avg=32768.0000 max=32963 max_rank=5 min=32574 min_rank=0 imbalance_ratio=1.005951 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=5 tokens_per_ep_rank=[32574, 32649, 32886, 32841, 32742, 32963, 32697, 32792] avg=32768.0000 max=32963 max_rank=5 min=32574 min_rank=0 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=4 tokens_per_ep_rank=[32704, 32535, 32850, 32985, 32655, 32710, 32866, 32839] avg=32768.0000 max=32985 max_rank=3 min=32535 min_rank=1 imbalance_ratio=1.006622 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=5 tokens_per_ep_rank=[32574, 32649, 32886, 32841, 32742, 32963, 32697, 32792] avg=32768.0000 max=32963 max_rank=5 min=32574 min_rank=0 imbalance_ratio=1.005951 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=4 tokens_per_ep_rank=[32704, 32535, 32850, 32985, 32655, 32710, 32866, 32839] avg=32768.0000 max=32985 max_rank=3 min=32535 min_rank=1 imbalance_ratio=1.006622 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=5 tokens_per_ep_rank=[32574, 32649, 32886, 32841, 32742, 32963, 32697, 32792] avg=32768.0000 max=32963 max_rank=5 min=32574 min_rank=0 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=5 tokens_per_ep_rank=[32574, 32649, 32886, 32841, 32742, 32963, 32697, 32792] avg=32768.0000 max=32963 max_rank=5 min=32574 min_rank=0 imbalance_ratio=1.005951 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=5 tokens_per_ep_rank=[32574, 32649, 32886, 32841, 32742, 32963, 32697, 32792] avg=32768.0000 max=32963 max_rank=5 min=32574 min_rank=0 imbalance_ratio=1.005951 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=5 tokens_per_ep_rank=[32574, 32649, 32886, 32841, 32742, 32963, 32697, 32792] avg=32768.0000 max=32963 max_rank=5 min=32574 min_rank=0 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=4 tokens_per_ep_rank=[32704, 32535, 32850, 32985, 32655, 32710, 32866, 32839] avg=32768.0000 max=32985 max_rank=3 min=32535 min_rank=1 imbalance_ratio=1.006622 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=5 tokens_per_ep_rank=[32574, 32649, 32886, 32841, 32742, 32963, 32697, 32792] avg=32768.0000 max=32963 max_rank=5 min=32574 min_rank=0 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=6 tokens_per_ep_rank=[33034, 32644, 33045, 32642, 32569, 32648, 32783, 32779] avg=32768.0000 max=33045 max_rank=2 min=32569 min_rank=4 imbalance_ratio=1.008453 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=6 tokens_per_ep_rank=[33034, 32644, 33045, 32642, 32569, 32648, 32783, 32779] avg=32768.0000 max=33045 max_rank=2 min=32569 min_rank=4 imbalance_ratio=1.008453 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=5 tokens_per_ep_rank=[32574, 32649, 32886, 32841, 32742, 32963, 32697, 32792] avg=32768.0000 max=32963 max_rank=5 min=32574 min_rank=0 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=5 tokens_per_ep_rank=[32574, 32649, 32886, 32841, 32742, 32963, 32697, 32792] avg=32768.0000 max=32963 max_rank=5 min=32574 min_rank=0 imbalance_ratio=1.005951 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=5 tokens_per_ep_rank=[32574, 32649, 32886, 32841, 32742, 32963, 32697, 32792] avg=32768.0000 max=32963 max_rank=5 min=32574 min_rank=0 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=6 tokens_per_ep_rank=[33034, 32644, 33045, 32642, 32569, 32648, 32783, 32779] avg=32768.0000 max=33045 max_rank=2 min=32569 min_rank=4 imbalance_ratio=1.008453 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=6 tokens_per_ep_rank=[33034, 32644, 33045, 32642, 32569, 32648, 32783, 32779] avg=32768.0000 max=33045 max_rank=2 min=32569 min_rank=4 imbalance_ratio=1.008453 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=6 tokens_per_ep_rank=[33034, 32644, 33045, 32642, 32569, 32648, 32783, 32779] avg=32768.0000 max=33045 max_rank=2 min=32569 min_rank=4 imbalance_ratio=1.008453 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=5 tokens_per_ep_rank=[32574, 32649, 32886, 32841, 32742, 32963, 32697, 32792] avg=32768.0000 max=32963 max_rank=5 min=32574 min_rank=0 imbalance_ratio=1.005951 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=6 tokens_per_ep_rank=[33034, 32644, 33045, 32642, 32569, 32648, 32783, 32779] avg=32768.0000 max=33045 max_rank=2 min=32569 min_rank=4 imbalance_ratio=1.008453 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=5 tokens_per_ep_rank=[32574, 32649, 32886, 32841, 32742, 32963, 32697, 32792] avg=32768.0000 max=32963 max_rank=5 min=32574 min_rank=0 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=6 tokens_per_ep_rank=[33034, 32644, 33045, 32642, 32569, 32648, 32783, 32779] avg=32768.0000 max=33045 max_rank=2 min=32569 min_rank=4 imbalance_ratio=1.008453 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=6 tokens_per_ep_rank=[33034, 32644, 33045, 32642, 32569, 32648, 32783, 32779] avg=32768.0000 max=33045 max_rank=2 min=32569 min_rank=4 imbalance_ratio=1.008453 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=6 tokens_per_ep_rank=[33034, 32644, 33045, 32642, 32569, 32648, 32783, 32779] avg=32768.0000 max=33045 max_rank=2 min=32569 min_rank=4 imbalance_ratio=1.008453 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=7 tokens_per_ep_rank=[32659, 32988, 32788, 32902, 32609, 32709, 32813, 32676] avg=32768.0000 max=32988 max_rank=1 min=32609 min_rank=4 imbalance_ratio=1.006714 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=5 tokens_per_ep_rank=[32574, 32649, 32886, 32841, 32742, 32963, 32697, 32792] avg=32768.0000 max=32963 max_rank=5 min=32574 min_rank=0 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=6 tokens_per_ep_rank=[33034, 32644, 33045, 32642, 32569, 32648, 32783, 32779] avg=32768.0000 max=33045 max_rank=2 min=32569 min_rank=4 imbalance_ratio=1.008453 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=7 tokens_per_ep_rank=[32659, 32988, 32788, 32902, 32609, 32709, 32813, 32676] avg=32768.0000 max=32988 max_rank=1 min=32609 min_rank=4 imbalance_ratio=1.006714 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=6 tokens_per_ep_rank=[33034, 32644, 33045, 32642, 32569, 32648, 32783, 32779] avg=32768.0000 max=33045 max_rank=2 min=32569 min_rank=4 imbalance_ratio=1.008453 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=6 tokens_per_ep_rank=[33034, 32644, 33045, 32642, 32569, 32648, 32783, 32779] avg=32768.0000 max=33045 max_rank=2 min=32569 min_rank=4 imbalance_ratio=1.008453 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=7 tokens_per_ep_rank=[32659, 32988, 32788, 32902, 32609, 32709, 32813, 32676] avg=32768.0000 max=32988 max_rank=1 min=32609 min_rank=4 imbalance_ratio=1.006714 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=6 tokens_per_ep_rank=[33034, 32644, 33045, 32642, 32569, 32648, 32783, 32779] avg=32768.0000 max=33045 max_rank=2 min=32569 min_rank=4 imbalance_ratio=1.008453 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=7 tokens_per_ep_rank=[32659, 32988, 32788, 32902, 32609, 32709, 32813, 32676] avg=32768.0000 max=32988 max_rank=1 min=32609 min_rank=4 imbalance_ratio=1.006714 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=7 tokens_per_ep_rank=[32659, 32988, 32788, 32902, 32609, 32709, 32813, 32676] avg=32768.0000 max=32988 max_rank=1 min=32609 min_rank=4 imbalance_ratio=1.006714 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=7 tokens_per_ep_rank=[32659, 32988, 32788, 32902, 32609, 32709, 32813, 32676] avg=32768.0000 max=32988 max_rank=1 min=32609 min_rank=4 imbalance_ratio=1.006714 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=6 tokens_per_ep_rank=[33034, 32644, 33045, 32642, 32569, 32648, 32783, 32779] avg=32768.0000 max=33045 max_rank=2 min=32569 min_rank=4 imbalance_ratio=1.008453 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=7 tokens_per_ep_rank=[32659, 32988, 32788, 32902, 32609, 32709, 32813, 32676] avg=32768.0000 max=32988 max_rank=1 min=32609 min_rank=4 imbalance_ratio=1.006714 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=6 tokens_per_ep_rank=[33034, 32644, 33045, 32642, 32569, 32648, 32783, 32779] avg=32768.0000 max=33045 max_rank=2 min=32569 min_rank=4 imbalance_ratio=1.008453 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=7 tokens_per_ep_rank=[32659, 32988, 32788, 32902, 32609, 32709, 32813, 32676] avg=32768.0000 max=32988 max_rank=1 min=32609 min_rank=4 imbalance_ratio=1.006714 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=7 tokens_per_ep_rank=[32659, 32988, 32788, 32902, 32609, 32709, 32813, 32676] avg=32768.0000 max=32988 max_rank=1 min=32609 min_rank=4 imbalance_ratio=1.006714 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=8 tokens_per_ep_rank=[32431, 32772, 32726, 32651, 32898, 32722, 32916, 33028] avg=32768.0000 max=33028 max_rank=7 min=32431 min_rank=0 imbalance_ratio=1.007935 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=7 tokens_per_ep_rank=[32659, 32988, 32788, 32902, 32609, 32709, 32813, 32676] avg=32768.0000 max=32988 max_rank=1 min=32609 min_rank=4 imbalance_ratio=1.006714 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=6 tokens_per_ep_rank=[33034, 32644, 33045, 32642, 32569, 32648, 32783, 32779] avg=32768.0000 max=33045 max_rank=2 min=32569 min_rank=4 imbalance_ratio=1.008453 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=8 tokens_per_ep_rank=[32431, 32772, 32726, 32651, 32898, 32722, 32916, 33028] avg=32768.0000 max=33028 max_rank=7 min=32431 min_rank=0 imbalance_ratio=1.007935 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=7 tokens_per_ep_rank=[32659, 32988, 32788, 32902, 32609, 32709, 32813, 32676] avg=32768.0000 max=32988 max_rank=1 min=32609 min_rank=4 imbalance_ratio=1.006714 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=7 tokens_per_ep_rank=[32659, 32988, 32788, 32902, 32609, 32709, 32813, 32676] avg=32768.0000 max=32988 max_rank=1 min=32609 min_rank=4 imbalance_ratio=1.006714 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=8 tokens_per_ep_rank=[32431, 32772, 32726, 32651, 32898, 32722, 32916, 33028] avg=32768.0000 max=33028 max_rank=7 min=32431 min_rank=0 imbalance_ratio=1.007935 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=7 tokens_per_ep_rank=[32659, 32988, 32788, 32902, 32609, 32709, 32813, 32676] avg=32768.0000 max=32988 max_rank=1 min=32609 min_rank=4 imbalance_ratio=1.006714 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=8 tokens_per_ep_rank=[32431, 32772, 32726, 32651, 32898, 32722, 32916, 33028] avg=32768.0000 max=33028 max_rank=7 min=32431 min_rank=0 imbalance_ratio=1.007935 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=8 tokens_per_ep_rank=[32431, 32772, 32726, 32651, 32898, 32722, 32916, 33028] avg=32768.0000 max=33028 max_rank=7 min=32431 min_rank=0 imbalance_ratio=1.007935 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=8 tokens_per_ep_rank=[32431, 32772, 32726, 32651, 32898, 32722, 32916, 33028] avg=32768.0000 max=33028 max_rank=7 min=32431 min_rank=0 imbalance_ratio=1.007935 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=7 tokens_per_ep_rank=[32659, 32988, 32788, 32902, 32609, 32709, 32813, 32676] avg=32768.0000 max=32988 max_rank=1 min=32609 min_rank=4 imbalance_ratio=1.006714 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=8 tokens_per_ep_rank=[32431, 32772, 32726, 32651, 32898, 32722, 32916, 33028] avg=32768.0000 max=33028 max_rank=7 min=32431 min_rank=0 imbalance_ratio=1.007935 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=7 tokens_per_ep_rank=[32659, 32988, 32788, 32902, 32609, 32709, 32813, 32676] avg=32768.0000 max=32988 max_rank=1 min=32609 min_rank=4 imbalance_ratio=1.006714 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=8 tokens_per_ep_rank=[32431, 32772, 32726, 32651, 32898, 32722, 32916, 33028] avg=32768.0000 max=33028 max_rank=7 min=32431 min_rank=0 imbalance_ratio=1.007935 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=8 tokens_per_ep_rank=[32431, 32772, 32726, 32651, 32898, 32722, 32916, 33028] avg=32768.0000 max=33028 max_rank=7 min=32431 min_rank=0 imbalance_ratio=1.007935 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=8 tokens_per_ep_rank=[32431, 32772, 32726, 32651, 32898, 32722, 32916, 33028] avg=32768.0000 max=33028 max_rank=7 min=32431 min_rank=0 imbalance_ratio=1.007935 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=9 tokens_per_ep_rank=[33067, 32796, 32871, 32745, 32745, 32504, 32612, 32804] avg=32768.0000 max=33067 max_rank=0 min=32504 min_rank=5 imbalance_ratio=1.009125 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=7 tokens_per_ep_rank=[32659, 32988, 32788, 32902, 32609, 32709, 32813, 32676] avg=32768.0000 max=32988 max_rank=1 min=32609 min_rank=4 imbalance_ratio=1.006714 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=9 tokens_per_ep_rank=[33067, 32796, 32871, 32745, 32745, 32504, 32612, 32804] avg=32768.0000 max=33067 max_rank=0 min=32504 min_rank=5 imbalance_ratio=1.009125 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=8 tokens_per_ep_rank=[32431, 32772, 32726, 32651, 32898, 32722, 32916, 33028] avg=32768.0000 max=33028 max_rank=7 min=32431 min_rank=0 imbalance_ratio=1.007935 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=9 tokens_per_ep_rank=[33067, 32796, 32871, 32745, 32745, 32504, 32612, 32804] avg=32768.0000 max=33067 max_rank=0 min=32504 min_rank=5 imbalance_ratio=1.009125 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=8 tokens_per_ep_rank=[32431, 32772, 32726, 32651, 32898, 32722, 32916, 33028] avg=32768.0000 max=33028 max_rank=7 min=32431 min_rank=0 imbalance_ratio=1.007935 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=9 tokens_per_ep_rank=[33067, 32796, 32871, 32745, 32745, 32504, 32612, 32804] avg=32768.0000 max=33067 max_rank=0 min=32504 min_rank=5 imbalance_ratio=1.009125 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=8 tokens_per_ep_rank=[32431, 32772, 32726, 32651, 32898, 32722, 32916, 33028] avg=32768.0000 max=33028 max_rank=7 min=32431 min_rank=0 imbalance_ratio=1.007935 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=9 tokens_per_ep_rank=[33067, 32796, 32871, 32745, 32745, 32504, 32612, 32804] avg=32768.0000 max=33067 max_rank=0 min=32504 min_rank=5 imbalance_ratio=1.009125 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=9 tokens_per_ep_rank=[33067, 32796, 32871, 32745, 32745, 32504, 32612, 32804] avg=32768.0000 max=33067 max_rank=0 min=32504 min_rank=5 imbalance_ratio=1.009125 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=8 tokens_per_ep_rank=[32431, 32772, 32726, 32651, 32898, 32722, 32916, 33028] avg=32768.0000 max=33028 max_rank=7 min=32431 min_rank=0 imbalance_ratio=1.007935 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=9 tokens_per_ep_rank=[33067, 32796, 32871, 32745, 32745, 32504, 32612, 32804] avg=32768.0000 max=33067 max_rank=0 min=32504 min_rank=5 imbalance_ratio=1.009125 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=9 tokens_per_ep_rank=[33067, 32796, 32871, 32745, 32745, 32504, 32612, 32804] avg=32768.0000 max=33067 max_rank=0 min=32504 min_rank=5 imbalance_ratio=1.009125 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=8 tokens_per_ep_rank=[32431, 32772, 32726, 32651, 32898, 32722, 32916, 33028] avg=32768.0000 max=33028 max_rank=7 min=32431 min_rank=0 imbalance_ratio=1.007935 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=10 tokens_per_ep_rank=[33051, 32723, 32667, 32642, 32866, 32562, 33173, 32460] avg=32768.0000 max=33173 max_rank=6 min=32460 min_rank=7 imbalance_ratio=1.012360 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=9 tokens_per_ep_rank=[33067, 32796, 32871, 32745, 32745, 32504, 32612, 32804] avg=32768.0000 max=33067 max_rank=0 min=32504 min_rank=5 imbalance_ratio=1.009125 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=9 tokens_per_ep_rank=[33067, 32796, 32871, 32745, 32745, 32504, 32612, 32804] avg=32768.0000 max=33067 max_rank=0 min=32504 min_rank=5 imbalance_ratio=1.009125 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=10 tokens_per_ep_rank=[33051, 32723, 32667, 32642, 32866, 32562, 33173, 32460] avg=32768.0000 max=33173 max_rank=6 min=32460 min_rank=7 imbalance_ratio=1.012360 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=9 tokens_per_ep_rank=[33067, 32796, 32871, 32745, 32745, 32504, 32612, 32804] avg=32768.0000 max=33067 max_rank=0 min=32504 min_rank=5 imbalance_ratio=1.009125 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=8 tokens_per_ep_rank=[32431, 32772, 32726, 32651, 32898, 32722, 32916, 33028] avg=32768.0000 max=33028 max_rank=7 min=32431 min_rank=0 imbalance_ratio=1.007935 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=10 tokens_per_ep_rank=[33051, 32723, 32667, 32642, 32866, 32562, 33096, 32537] avg=32768.0000 max=33096 max_rank=6 min=32537 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=77 max_load_reduction_pct=0.002321 moved_to_replicas=77 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=9 tokens_per_ep_rank=[33067, 32796, 32871, 32745, 32745, 32504, 32612, 32804] avg=32768.0000 max=33067 max_rank=0 min=32504 min_rank=5 imbalance_ratio=1.009125 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=10 tokens_per_ep_rank=[33051, 32723, 32667, 32642, 32866, 32562, 33173, 32460] avg=32768.0000 max=33173 max_rank=6 min=32460 min_rank=7 imbalance_ratio=1.012360 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=9 tokens_per_ep_rank=[33067, 32796, 32871, 32745, 32745, 32504, 32612, 32804] avg=32768.0000 max=33067 max_rank=0 min=32504 min_rank=5 imbalance_ratio=1.009125 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=10 tokens_per_ep_rank=[33051, 32723, 32667, 32642, 32866, 32562, 33173, 32460] avg=32768.0000 max=33173 max_rank=6 min=32460 min_rank=7 imbalance_ratio=1.012360 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=10 tokens_per_ep_rank=[33051, 32723, 32667, 32642, 32866, 32562, 33096, 32537] avg=32768.0000 max=33096 max_rank=6 min=32537 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=77 max_load_reduction_pct=0.002321 moved_to_replicas=77 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=10 tokens_per_ep_rank=[33051, 32723, 32667, 32642, 32866, 32562, 33173, 32460] avg=32768.0000 max=33173 max_rank=6 min=32460 min_rank=7 imbalance_ratio=1.012360 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=10 tokens_per_ep_rank=[33051, 32723, 32667, 32642, 32866, 32562, 33096, 32537] avg=32768.0000 max=33096 max_rank=6 min=32537 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=77 max_load_reduction_pct=0.002321 moved_to_replicas=77 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=9 tokens_per_ep_rank=[33067, 32796, 32871, 32745, 32745, 32504, 32612, 32804] avg=32768.0000 max=33067 max_rank=0 min=32504 min_rank=5 imbalance_ratio=1.009125 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=9 tokens_per_ep_rank=[33067, 32796, 32871, 32745, 32745, 32504, 32612, 32804] avg=32768.0000 max=33067 max_rank=0 min=32504 min_rank=5 imbalance_ratio=1.009125 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=10 tokens_per_ep_rank=[33051, 32723, 32667, 32642, 32866, 32562, 33096, 32537] avg=32768.0000 max=33096 max_rank=6 min=32537 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=77 max_load_reduction_pct=0.002321 moved_to_replicas=77 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=10 tokens_per_ep_rank=[33051, 32723, 32667, 32642, 32866, 32562, 33173, 32460] avg=32768.0000 max=33173 max_rank=6 min=32460 min_rank=7 imbalance_ratio=1.012360 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=10 tokens_per_ep_rank=[33051, 32723, 32667, 32642, 32866, 32562, 33096, 32537] avg=32768.0000 max=33096 max_rank=6 min=32537 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=77 max_load_reduction_pct=0.002321 moved_to_replicas=77 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=9 tokens_per_ep_rank=[33067, 32796, 32871, 32745, 32745, 32504, 32612, 32804] avg=32768.0000 max=33067 max_rank=0 min=32504 min_rank=5 imbalance_ratio=1.009125 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=10 tokens_per_ep_rank=[33051, 32723, 32667, 32642, 32866, 32562, 33096, 32537] avg=32768.0000 max=33096 max_rank=6 min=32537 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=77 max_load_reduction_pct=0.002321 moved_to_replicas=77 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=10 tokens_per_ep_rank=[33051, 32723, 32667, 32642, 32866, 32562, 33173, 32460] avg=32768.0000 max=33173 max_rank=6 min=32460 min_rank=7 imbalance_ratio=1.012360 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=10 tokens_per_ep_rank=[33051, 32723, 32667, 32642, 32866, 32562, 33173, 32460] avg=32768.0000 max=33173 max_rank=6 min=32460 min_rank=7 imbalance_ratio=1.012360 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=10 tokens_per_ep_rank=[33051, 32723, 32667, 32642, 32866, 32562, 33096, 32537] avg=32768.0000 max=33096 max_rank=6 min=32537 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=77 max_load_reduction_pct=0.002321 moved_to_replicas=77 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=10 tokens_per_ep_rank=[33051, 32723, 32667, 32642, 32866, 32562, 33096, 32537] avg=32768.0000 max=33096 max_rank=6 min=32537 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=77 max_load_reduction_pct=0.002321 moved_to_replicas=77 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=8 records=10 total_expected_recv=326646 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=78592 peak_demand=32983 required_capacity=1.006561 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=8 records=10 total_expected_recv=328087 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76544 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=8 records=10 total_expected_recv=328203 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76544 peak_demand=33045 required_capacity=1.008453 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=8 records=10 total_expected_recv=328385 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76288 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=8 records=10 total_expected_recv=326827 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=78464 peak_demand=32898 required_capacity=1.003967 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=8 records=10 total_expected_recv=327793 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=75904 peak_demand=33078 required_capacity=1.009460 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=8 records=10 total_expected_recv=327786 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76288 peak_demand=33067 required_capacity=1.009125 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=8 records=10 total_expected_recv=327713 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76544 peak_demand=32985 required_capacity=1.006622 + [2026-08-26 15:53:13] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 212.4 | throughput per GPU (TFLOP/s/GPU): 581.3 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.617200E+00 | loss scale: 1.0 | grad norm: 3.963 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 33664 | min safe capacity: 1.0273 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=0 calls=1 budget=40960 expected_recv=32432 expected_overflow=0 exact_recv_per_rank=[32730, 32680, 32954, 32876, 32652, 32432, 32821, 32999] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=0 calls=1 budget=40960 expected_recv=32954 expected_overflow=0 exact_recv_per_rank=[32730, 32680, 32954, 32876, 32652, 32432, 32821, 32999] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=0 calls=1 budget=40960 expected_recv=32821 expected_overflow=0 exact_recv_per_rank=[32730, 32680, 32954, 32876, 32652, 32432, 32821, 32999] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=0 calls=1 budget=40960 expected_recv=32876 expected_overflow=0 exact_recv_per_rank=[32730, 32680, 32954, 32876, 32652, 32432, 32821, 32999] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=0 calls=1 budget=40960 expected_recv=32652 expected_overflow=0 exact_recv_per_rank=[32730, 32680, 32954, 32876, 32652, 32432, 32821, 32999] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=0 calls=1 budget=40960 expected_recv=32680 expected_overflow=0 exact_recv_per_rank=[32730, 32680, 32954, 32876, 32652, 32432, 32821, 32999] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=0 calls=1 budget=40960 expected_recv=32999 expected_overflow=0 exact_recv_per_rank=[32730, 32680, 32954, 32876, 32652, 32432, 32821, 32999] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=0 calls=1 budget=40960 expected_recv=32432 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=0 calls=1 budget=40960 expected_recv=32954 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=0 calls=1 budget=40960 expected_recv=32730 expected_overflow=0 exact_recv_per_rank=[32730, 32680, 32954, 32876, 32652, 32432, 32821, 32999] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=0 calls=1 budget=40960 expected_recv=32821 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=0 calls=1 budget=40960 expected_recv=32876 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=0 calls=1 budget=40960 expected_recv=32652 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=0 calls=1 budget=40960 expected_recv=32680 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=0 calls=1 budget=40960 expected_recv=32999 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=0 calls=1 budget=40960 expected_recv=32730 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=1 calls=1 budget=40960 expected_recv=32693 expected_overflow=0 exact_recv_per_rank=[32579, 32879, 32866, 32870, 32722, 32693, 32764, 32771] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=1 calls=1 budget=40960 expected_recv=32866 expected_overflow=0 exact_recv_per_rank=[32579, 32879, 32866, 32870, 32722, 32693, 32764, 32771] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=1 calls=1 budget=40960 expected_recv=32764 expected_overflow=0 exact_recv_per_rank=[32579, 32879, 32866, 32870, 32722, 32693, 32764, 32771] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=1 calls=1 budget=40960 expected_recv=32870 expected_overflow=0 exact_recv_per_rank=[32579, 32879, 32866, 32870, 32722, 32693, 32764, 32771] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=1 calls=1 budget=40960 expected_recv=32693 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=1 calls=1 budget=40960 expected_recv=32866 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=1 calls=1 budget=40960 expected_recv=32722 expected_overflow=0 exact_recv_per_rank=[32579, 32879, 32866, 32870, 32722, 32693, 32764, 32771] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=1 calls=1 budget=40960 expected_recv=32764 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=1 calls=1 budget=40960 expected_recv=32879 expected_overflow=0 exact_recv_per_rank=[32579, 32879, 32866, 32870, 32722, 32693, 32764, 32771] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=1 calls=1 budget=40960 expected_recv=32771 expected_overflow=0 exact_recv_per_rank=[32579, 32879, 32866, 32870, 32722, 32693, 32764, 32771] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=1 calls=1 budget=40960 expected_recv=32870 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=1 calls=1 budget=40960 expected_recv=32722 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=1 calls=1 budget=40960 expected_recv=32579 expected_overflow=0 exact_recv_per_rank=[32579, 32879, 32866, 32870, 32722, 32693, 32764, 32771] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=1 calls=1 budget=40960 expected_recv=32879 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=1 calls=1 budget=40960 expected_recv=32771 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=1 calls=1 budget=40960 expected_recv=32579 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=2 calls=1 budget=40960 expected_recv=32710 expected_overflow=0 exact_recv_per_rank=[32603, 33022, 32872, 32770, 32851, 32710, 32678, 32638] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=2 calls=1 budget=40960 expected_recv=32872 expected_overflow=0 exact_recv_per_rank=[32603, 33022, 32872, 32770, 32851, 32710, 32678, 32638] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=2 calls=1 budget=40960 expected_recv=32710 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=2 calls=1 budget=40960 expected_recv=32872 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=2 calls=1 budget=40960 expected_recv=32678 expected_overflow=0 exact_recv_per_rank=[32603, 33022, 32872, 32770, 32851, 32710, 32678, 32638] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=2 calls=1 budget=40960 expected_recv=32678 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=2 calls=1 budget=40960 expected_recv=32851 expected_overflow=0 exact_recv_per_rank=[32603, 33022, 32872, 32770, 32851, 32710, 32678, 32638] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=2 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[32603, 33022, 32872, 32770, 32851, 32710, 32678, 32638] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=2 calls=1 budget=40960 expected_recv=33022 expected_overflow=0 exact_recv_per_rank=[32603, 33022, 32872, 32770, 32851, 32710, 32678, 32638] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=2 calls=1 budget=40960 expected_recv=32638 expected_overflow=0 exact_recv_per_rank=[32603, 33022, 32872, 32770, 32851, 32710, 32678, 32638] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=2 calls=1 budget=40960 expected_recv=32851 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=2 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=2 calls=1 budget=40960 expected_recv=33022 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=2 calls=1 budget=40960 expected_recv=32638 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=2 calls=1 budget=40960 expected_recv=32603 expected_overflow=0 exact_recv_per_rank=[32603, 33022, 32872, 32770, 32851, 32710, 32678, 32638] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=3 calls=1 budget=40960 expected_recv=32845 expected_overflow=0 exact_recv_per_rank=[32943, 32854, 32704, 32700, 32784, 32845, 32505, 32809] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=2 calls=1 budget=40960 expected_recv=32603 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=3 calls=1 budget=40960 expected_recv=32704 expected_overflow=0 exact_recv_per_rank=[32943, 32854, 32704, 32700, 32784, 32845, 32505, 32809] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=3 calls=1 budget=40960 expected_recv=32845 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=3 calls=1 budget=40960 expected_recv=32704 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=3 calls=1 budget=40960 expected_recv=32505 expected_overflow=0 exact_recv_per_rank=[32943, 32854, 32704, 32700, 32784, 32845, 32505, 32809] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=3 calls=1 budget=40960 expected_recv=32505 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=3 calls=1 budget=40960 expected_recv=32784 expected_overflow=0 exact_recv_per_rank=[32943, 32854, 32704, 32700, 32784, 32845, 32505, 32809] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=3 calls=1 budget=40960 expected_recv=32700 expected_overflow=0 exact_recv_per_rank=[32943, 32854, 32704, 32700, 32784, 32845, 32505, 32809] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=3 calls=1 budget=40960 expected_recv=32854 expected_overflow=0 exact_recv_per_rank=[32943, 32854, 32704, 32700, 32784, 32845, 32505, 32809] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=3 calls=1 budget=40960 expected_recv=32809 expected_overflow=0 exact_recv_per_rank=[32943, 32854, 32704, 32700, 32784, 32845, 32505, 32809] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=3 calls=1 budget=40960 expected_recv=32784 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=3 calls=1 budget=40960 expected_recv=32700 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=3 calls=1 budget=40960 expected_recv=32943 expected_overflow=0 exact_recv_per_rank=[32943, 32854, 32704, 32700, 32784, 32845, 32505, 32809] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=3 calls=1 budget=40960 expected_recv=32854 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=3 calls=1 budget=40960 expected_recv=32809 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=4 calls=1 budget=40960 expected_recv=32888 expected_overflow=0 exact_recv_per_rank=[32710, 32846, 32787, 32620, 32929, 32888, 32594, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=3 calls=1 budget=40960 expected_recv=32943 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=4 calls=1 budget=40960 expected_recv=32787 expected_overflow=0 exact_recv_per_rank=[32710, 32846, 32787, 32620, 32929, 32888, 32594, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=4 calls=1 budget=40960 expected_recv=32888 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=4 calls=1 budget=40960 expected_recv=32787 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=4 calls=1 budget=40960 expected_recv=32594 expected_overflow=0 exact_recv_per_rank=[32710, 32846, 32787, 32620, 32929, 32888, 32594, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=4 calls=1 budget=40960 expected_recv=32594 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=4 calls=1 budget=40960 expected_recv=32929 expected_overflow=0 exact_recv_per_rank=[32710, 32846, 32787, 32620, 32929, 32888, 32594, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=4 calls=1 budget=40960 expected_recv=32846 expected_overflow=0 exact_recv_per_rank=[32710, 32846, 32787, 32620, 32929, 32888, 32594, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=4 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[32710, 32846, 32787, 32620, 32929, 32888, 32594, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=4 calls=1 budget=40960 expected_recv=32620 expected_overflow=0 exact_recv_per_rank=[32710, 32846, 32787, 32620, 32929, 32888, 32594, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=4 calls=1 budget=40960 expected_recv=32929 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=4 calls=1 budget=40960 expected_recv=32710 expected_overflow=0 exact_recv_per_rank=[32710, 32846, 32787, 32620, 32929, 32888, 32594, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=4 calls=1 budget=40960 expected_recv=32846 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=4 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=4 calls=1 budget=40960 expected_recv=32620 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=4 calls=1 budget=40960 expected_recv=32710 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=5 calls=1 budget=40960 expected_recv=32843 expected_overflow=0 exact_recv_per_rank=[32631, 32920, 32810, 32290, 33041, 32843, 32787, 32822] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=5 calls=1 budget=40960 expected_recv=32810 expected_overflow=0 exact_recv_per_rank=[32631, 32920, 32810, 32290, 33041, 32843, 32787, 32822] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=5 calls=1 budget=40960 expected_recv=32843 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=5 calls=1 budget=40960 expected_recv=32787 expected_overflow=0 exact_recv_per_rank=[32631, 32920, 32810, 32290, 33041, 32843, 32787, 32822] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=5 calls=1 budget=40960 expected_recv=32810 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=5 calls=1 budget=40960 expected_recv=32787 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=5 calls=1 budget=40960 expected_recv=33041 expected_overflow=0 exact_recv_per_rank=[32631, 32920, 32810, 32290, 33041, 32843, 32787, 32822] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=5 calls=1 budget=40960 expected_recv=32920 expected_overflow=0 exact_recv_per_rank=[32631, 32920, 32810, 32290, 33041, 32843, 32787, 32822] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=5 calls=1 budget=40960 expected_recv=32822 expected_overflow=0 exact_recv_per_rank=[32631, 32920, 32810, 32290, 33041, 32843, 32787, 32822] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=5 calls=1 budget=40960 expected_recv=33041 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=5 calls=1 budget=40960 expected_recv=32920 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=5 calls=1 budget=40960 expected_recv=32822 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=5 calls=1 budget=40960 expected_recv=32631 expected_overflow=0 exact_recv_per_rank=[32631, 32920, 32810, 32290, 33041, 32843, 32787, 32822] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=5 calls=1 budget=40960 expected_recv=32290 expected_overflow=0 exact_recv_per_rank=[32631, 32920, 32810, 32290, 33041, 32843, 32787, 32822] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=5 calls=1 budget=40960 expected_recv=32631 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=6 calls=1 budget=40960 expected_recv=32726 expected_overflow=0 exact_recv_per_rank=[32889, 32708, 33070, 32588, 32656, 32726, 32782, 32725] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=6 calls=1 budget=40960 expected_recv=33070 expected_overflow=0 exact_recv_per_rank=[32889, 32708, 33070, 32588, 32656, 32726, 32782, 32725] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=5 calls=1 budget=40960 expected_recv=32290 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=6 calls=1 budget=40960 expected_recv=32726 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=6 calls=1 budget=40960 expected_recv=32782 expected_overflow=0 exact_recv_per_rank=[32889, 32708, 33070, 32588, 32656, 32726, 32782, 32725] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=6 calls=1 budget=40960 expected_recv=33070 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=6 calls=1 budget=40960 expected_recv=32782 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=6 calls=1 budget=40960 expected_recv=32656 expected_overflow=0 exact_recv_per_rank=[32889, 32708, 33070, 32588, 32656, 32726, 32782, 32725] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=6 calls=1 budget=40960 expected_recv=32656 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=6 calls=1 budget=40960 expected_recv=32708 expected_overflow=0 exact_recv_per_rank=[32889, 32708, 33070, 32588, 32656, 32726, 32782, 32725] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=6 calls=1 budget=40960 expected_recv=32725 expected_overflow=0 exact_recv_per_rank=[32889, 32708, 33070, 32588, 32656, 32726, 32782, 32725] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=6 calls=1 budget=40960 expected_recv=32708 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=6 calls=1 budget=40960 expected_recv=32889 expected_overflow=0 exact_recv_per_rank=[32889, 32708, 33070, 32588, 32656, 32726, 32782, 32725] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=6 calls=1 budget=40960 expected_recv=32725 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=6 calls=1 budget=40960 expected_recv=32588 expected_overflow=0 exact_recv_per_rank=[32889, 32708, 33070, 32588, 32656, 32726, 32782, 32725] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=7 calls=1 budget=40960 expected_recv=32836 expected_overflow=0 exact_recv_per_rank=[32606, 32773, 32885, 32820, 32616, 32836, 32857, 32751] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=6 calls=1 budget=40960 expected_recv=32889 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=7 calls=1 budget=40960 expected_recv=32885 expected_overflow=0 exact_recv_per_rank=[32606, 32773, 32885, 32820, 32616, 32836, 32857, 32751] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=6 calls=1 budget=40960 expected_recv=32588 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=7 calls=1 budget=40960 expected_recv=32836 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=7 calls=1 budget=40960 expected_recv=32857 expected_overflow=0 exact_recv_per_rank=[32606, 32773, 32885, 32820, 32616, 32836, 32857, 32751] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=7 calls=1 budget=40960 expected_recv=32885 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=7 calls=1 budget=40960 expected_recv=32616 expected_overflow=0 exact_recv_per_rank=[32606, 32773, 32885, 32820, 32616, 32836, 32857, 32751] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=7 calls=1 budget=40960 expected_recv=32857 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=7 calls=1 budget=40960 expected_recv=32616 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=7 calls=1 budget=40960 expected_recv=32773 expected_overflow=0 exact_recv_per_rank=[32606, 32773, 32885, 32820, 32616, 32836, 32857, 32751] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=7 calls=1 budget=40960 expected_recv=32751 expected_overflow=0 exact_recv_per_rank=[32606, 32773, 32885, 32820, 32616, 32836, 32857, 32751] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=7 calls=1 budget=40960 expected_recv=32773 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=7 calls=1 budget=40960 expected_recv=32751 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=7 calls=1 budget=40960 expected_recv=32606 expected_overflow=0 exact_recv_per_rank=[32606, 32773, 32885, 32820, 32616, 32836, 32857, 32751] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=7 calls=1 budget=40960 expected_recv=32820 expected_overflow=0 exact_recv_per_rank=[32606, 32773, 32885, 32820, 32616, 32836, 32857, 32751] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=8 calls=1 budget=40960 expected_recv=32808 expected_overflow=0 exact_recv_per_rank=[32791, 32682, 32627, 33096, 32803, 32808, 32769, 32568] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=7 calls=1 budget=40960 expected_recv=32820 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=8 calls=1 budget=40960 expected_recv=32627 expected_overflow=0 exact_recv_per_rank=[32791, 32682, 32627, 33096, 32803, 32808, 32769, 32568] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=7 calls=1 budget=40960 expected_recv=32606 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=8 calls=1 budget=40960 expected_recv=32808 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=8 calls=1 budget=40960 expected_recv=32627 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=8 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[32791, 32682, 32627, 33096, 32803, 32808, 32769, 32568] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=8 calls=1 budget=40960 expected_recv=32803 expected_overflow=0 exact_recv_per_rank=[32791, 32682, 32627, 33096, 32803, 32808, 32769, 32568] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=8 calls=1 budget=40960 expected_recv=32682 expected_overflow=0 exact_recv_per_rank=[32791, 32682, 32627, 33096, 32803, 32808, 32769, 32568] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=8 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=8 calls=1 budget=40960 expected_recv=32803 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=8 calls=1 budget=40960 expected_recv=32568 expected_overflow=0 exact_recv_per_rank=[32791, 32682, 32627, 33096, 32803, 32808, 32769, 32568] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=8 calls=1 budget=40960 expected_recv=32682 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32791, 32682, 32627, 33096, 32803, 32808, 32769, 32568] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=8 calls=1 budget=40960 expected_recv=32568 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=9 calls=1 budget=40960 expected_recv=32824 expected_overflow=0 exact_recv_per_rank=[32753, 32782, 32806, 32728, 32939, 32824, 32774, 32538] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=9 calls=1 budget=40960 expected_recv=32806 expected_overflow=0 exact_recv_per_rank=[32753, 32782, 32806, 32728, 32939, 32824, 32774, 32538] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=8 calls=1 budget=40960 expected_recv=32791 expected_overflow=0 exact_recv_per_rank=[32791, 32682, 32627, 33096, 32803, 32808, 32769, 32568] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=9 calls=1 budget=40960 expected_recv=32824 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=9 calls=1 budget=40960 expected_recv=32806 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=8 calls=1 budget=40960 expected_recv=32791 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=9 calls=1 budget=40960 expected_recv=32774 expected_overflow=0 exact_recv_per_rank=[32753, 32782, 32806, 32728, 32939, 32824, 32774, 32538] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=9 calls=1 budget=40960 expected_recv=32939 expected_overflow=0 exact_recv_per_rank=[32753, 32782, 32806, 32728, 32939, 32824, 32774, 32538] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=9 calls=1 budget=40960 expected_recv=32782 expected_overflow=0 exact_recv_per_rank=[32753, 32782, 32806, 32728, 32939, 32824, 32774, 32538] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=9 calls=1 budget=40960 expected_recv=32774 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=9 calls=1 budget=40960 expected_recv=32939 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=9 calls=1 budget=40960 expected_recv=32538 expected_overflow=0 exact_recv_per_rank=[32753, 32782, 32806, 32728, 32939, 32824, 32774, 32538] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=9 calls=1 budget=40960 expected_recv=32782 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=9 calls=1 budget=40960 expected_recv=32728 expected_overflow=0 exact_recv_per_rank=[32753, 32782, 32806, 32728, 32939, 32824, 32774, 32538] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=9 calls=1 budget=40960 expected_recv=32538 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=9 calls=1 budget=40960 expected_recv=32728 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=1 tokens_per_ep_rank=[32730, 32680, 32954, 32876, 32652, 32432, 32821, 32999] avg=32768.0000 max=32999 max_rank=7 min=32432 min_rank=5 imbalance_ratio=1.007050 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=9 calls=1 budget=40960 expected_recv=32753 expected_overflow=0 exact_recv_per_rank=[32753, 32782, 32806, 32728, 32939, 32824, 32774, 32538] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=1 tokens_per_ep_rank=[32730, 32680, 32954, 32876, 32652, 32432, 32821, 32999] avg=32768.0000 max=32999 max_rank=7 min=32432 min_rank=5 imbalance_ratio=1.007050 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=9 calls=1 budget=40960 expected_recv=32753 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=1 tokens_per_ep_rank=[32730, 32680, 32954, 32876, 32652, 32432, 32821, 32999] avg=32768.0000 max=32999 max_rank=7 min=32432 min_rank=5 imbalance_ratio=1.007050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=1 tokens_per_ep_rank=[32730, 32680, 32954, 32876, 32652, 32432, 32821, 32999] avg=32768.0000 max=32999 max_rank=7 min=32432 min_rank=5 imbalance_ratio=1.007050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=1 tokens_per_ep_rank=[32730, 32680, 32954, 32876, 32652, 32432, 32821, 32999] avg=32768.0000 max=32999 max_rank=7 min=32432 min_rank=5 imbalance_ratio=1.007050 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=1 tokens_per_ep_rank=[32730, 32680, 32954, 32876, 32652, 32432, 32821, 32999] avg=32768.0000 max=32999 max_rank=7 min=32432 min_rank=5 imbalance_ratio=1.007050 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=1 tokens_per_ep_rank=[32730, 32680, 32954, 32876, 32652, 32432, 32821, 32999] avg=32768.0000 max=32999 max_rank=7 min=32432 min_rank=5 imbalance_ratio=1.007050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=1 tokens_per_ep_rank=[32730, 32680, 32954, 32876, 32652, 32432, 32821, 32999] avg=32768.0000 max=32999 max_rank=7 min=32432 min_rank=5 imbalance_ratio=1.007050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=1 tokens_per_ep_rank=[32730, 32680, 32954, 32876, 32652, 32432, 32821, 32999] avg=32768.0000 max=32999 max_rank=7 min=32432 min_rank=5 imbalance_ratio=1.007050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=1 tokens_per_ep_rank=[32730, 32680, 32954, 32876, 32652, 32432, 32821, 32999] avg=32768.0000 max=32999 max_rank=7 min=32432 min_rank=5 imbalance_ratio=1.007050 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=1 tokens_per_ep_rank=[32730, 32680, 32954, 32876, 32652, 32432, 32821, 32999] avg=32768.0000 max=32999 max_rank=7 min=32432 min_rank=5 imbalance_ratio=1.007050 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=1 tokens_per_ep_rank=[32730, 32680, 32954, 32876, 32652, 32432, 32821, 32999] avg=32768.0000 max=32999 max_rank=7 min=32432 min_rank=5 imbalance_ratio=1.007050 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=2 tokens_per_ep_rank=[32579, 32879, 32866, 32870, 32722, 32693, 32764, 32771] avg=32768.0000 max=32879 max_rank=1 min=32579 min_rank=0 imbalance_ratio=1.003387 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=1 tokens_per_ep_rank=[32730, 32680, 32954, 32876, 32652, 32432, 32821, 32999] avg=32768.0000 max=32999 max_rank=7 min=32432 min_rank=5 imbalance_ratio=1.007050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=1 tokens_per_ep_rank=[32730, 32680, 32954, 32876, 32652, 32432, 32821, 32999] avg=32768.0000 max=32999 max_rank=7 min=32432 min_rank=5 imbalance_ratio=1.007050 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=2 tokens_per_ep_rank=[32579, 32879, 32866, 32870, 32722, 32693, 32764, 32771] avg=32768.0000 max=32879 max_rank=1 min=32579 min_rank=0 imbalance_ratio=1.003387 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=1 tokens_per_ep_rank=[32730, 32680, 32954, 32876, 32652, 32432, 32821, 32999] avg=32768.0000 max=32999 max_rank=7 min=32432 min_rank=5 imbalance_ratio=1.007050 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=2 tokens_per_ep_rank=[32579, 32879, 32866, 32870, 32722, 32693, 32764, 32771] avg=32768.0000 max=32879 max_rank=1 min=32579 min_rank=0 imbalance_ratio=1.003387 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=2 tokens_per_ep_rank=[32579, 32879, 32866, 32870, 32722, 32693, 32764, 32771] avg=32768.0000 max=32879 max_rank=1 min=32579 min_rank=0 imbalance_ratio=1.003387 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=2 tokens_per_ep_rank=[32579, 32879, 32866, 32870, 32722, 32693, 32764, 32771] avg=32768.0000 max=32879 max_rank=1 min=32579 min_rank=0 imbalance_ratio=1.003387 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=2 tokens_per_ep_rank=[32579, 32879, 32866, 32870, 32722, 32693, 32764, 32771] avg=32768.0000 max=32879 max_rank=1 min=32579 min_rank=0 imbalance_ratio=1.003387 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=1 tokens_per_ep_rank=[32730, 32680, 32954, 32876, 32652, 32432, 32821, 32999] avg=32768.0000 max=32999 max_rank=7 min=32432 min_rank=5 imbalance_ratio=1.007050 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=2 tokens_per_ep_rank=[32579, 32879, 32866, 32870, 32722, 32693, 32764, 32771] avg=32768.0000 max=32879 max_rank=1 min=32579 min_rank=0 imbalance_ratio=1.003387 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=2 tokens_per_ep_rank=[32579, 32879, 32866, 32870, 32722, 32693, 32764, 32771] avg=32768.0000 max=32879 max_rank=1 min=32579 min_rank=0 imbalance_ratio=1.003387 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=2 tokens_per_ep_rank=[32579, 32879, 32866, 32870, 32722, 32693, 32764, 32771] avg=32768.0000 max=32879 max_rank=1 min=32579 min_rank=0 imbalance_ratio=1.003387 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=2 tokens_per_ep_rank=[32579, 32879, 32866, 32870, 32722, 32693, 32764, 32771] avg=32768.0000 max=32879 max_rank=1 min=32579 min_rank=0 imbalance_ratio=1.003387 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=2 tokens_per_ep_rank=[32579, 32879, 32866, 32870, 32722, 32693, 32764, 32771] avg=32768.0000 max=32879 max_rank=1 min=32579 min_rank=0 imbalance_ratio=1.003387 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=3 tokens_per_ep_rank=[32603, 33022, 32872, 32770, 32851, 32710, 32678, 32638] avg=32768.0000 max=33022 max_rank=1 min=32603 min_rank=0 imbalance_ratio=1.007751 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=2 tokens_per_ep_rank=[32579, 32879, 32866, 32870, 32722, 32693, 32764, 32771] avg=32768.0000 max=32879 max_rank=1 min=32579 min_rank=0 imbalance_ratio=1.003387 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=2 tokens_per_ep_rank=[32579, 32879, 32866, 32870, 32722, 32693, 32764, 32771] avg=32768.0000 max=32879 max_rank=1 min=32579 min_rank=0 imbalance_ratio=1.003387 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=3 tokens_per_ep_rank=[32603, 33022, 32872, 32770, 32851, 32710, 32678, 32638] avg=32768.0000 max=33022 max_rank=1 min=32603 min_rank=0 imbalance_ratio=1.007751 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=2 tokens_per_ep_rank=[32579, 32879, 32866, 32870, 32722, 32693, 32764, 32771] avg=32768.0000 max=32879 max_rank=1 min=32579 min_rank=0 imbalance_ratio=1.003387 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=2 tokens_per_ep_rank=[32579, 32879, 32866, 32870, 32722, 32693, 32764, 32771] avg=32768.0000 max=32879 max_rank=1 min=32579 min_rank=0 imbalance_ratio=1.003387 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=3 tokens_per_ep_rank=[32603, 33022, 32872, 32770, 32851, 32710, 32678, 32638] avg=32768.0000 max=33022 max_rank=1 min=32603 min_rank=0 imbalance_ratio=1.007751 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=3 tokens_per_ep_rank=[32603, 33022, 32872, 32770, 32851, 32710, 32678, 32638] avg=32768.0000 max=33022 max_rank=1 min=32603 min_rank=0 imbalance_ratio=1.007751 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=3 tokens_per_ep_rank=[32603, 33022, 32872, 32770, 32851, 32710, 32678, 32638] avg=32768.0000 max=33022 max_rank=1 min=32603 min_rank=0 imbalance_ratio=1.007751 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=3 tokens_per_ep_rank=[32603, 33022, 32872, 32770, 32851, 32710, 32678, 32638] avg=32768.0000 max=33022 max_rank=1 min=32603 min_rank=0 imbalance_ratio=1.007751 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=2 tokens_per_ep_rank=[32579, 32879, 32866, 32870, 32722, 32693, 32764, 32771] avg=32768.0000 max=32879 max_rank=1 min=32579 min_rank=0 imbalance_ratio=1.003387 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=3 tokens_per_ep_rank=[32603, 33022, 32872, 32770, 32851, 32710, 32678, 32638] avg=32768.0000 max=33022 max_rank=1 min=32603 min_rank=0 imbalance_ratio=1.007751 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=3 tokens_per_ep_rank=[32603, 33022, 32872, 32770, 32851, 32710, 32678, 32638] avg=32768.0000 max=33022 max_rank=1 min=32603 min_rank=0 imbalance_ratio=1.007751 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=3 tokens_per_ep_rank=[32603, 33022, 32872, 32770, 32851, 32710, 32678, 32638] avg=32768.0000 max=33022 max_rank=1 min=32603 min_rank=0 imbalance_ratio=1.007751 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=3 tokens_per_ep_rank=[32603, 33022, 32872, 32770, 32851, 32710, 32678, 32638] avg=32768.0000 max=33022 max_rank=1 min=32603 min_rank=0 imbalance_ratio=1.007751 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=3 tokens_per_ep_rank=[32603, 33022, 32872, 32770, 32851, 32710, 32678, 32638] avg=32768.0000 max=33022 max_rank=1 min=32603 min_rank=0 imbalance_ratio=1.007751 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=4 tokens_per_ep_rank=[32943, 32854, 32704, 32700, 32784, 32845, 32505, 32809] avg=32768.0000 max=32943 max_rank=0 min=32505 min_rank=6 imbalance_ratio=1.005341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=3 tokens_per_ep_rank=[32603, 33022, 32872, 32770, 32851, 32710, 32678, 32638] avg=32768.0000 max=33022 max_rank=1 min=32603 min_rank=0 imbalance_ratio=1.007751 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=3 tokens_per_ep_rank=[32603, 33022, 32872, 32770, 32851, 32710, 32678, 32638] avg=32768.0000 max=33022 max_rank=1 min=32603 min_rank=0 imbalance_ratio=1.007751 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=4 tokens_per_ep_rank=[32943, 32854, 32704, 32700, 32784, 32845, 32505, 32809] avg=32768.0000 max=32943 max_rank=0 min=32505 min_rank=6 imbalance_ratio=1.005341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=3 tokens_per_ep_rank=[32603, 33022, 32872, 32770, 32851, 32710, 32678, 32638] avg=32768.0000 max=33022 max_rank=1 min=32603 min_rank=0 imbalance_ratio=1.007751 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=3 tokens_per_ep_rank=[32603, 33022, 32872, 32770, 32851, 32710, 32678, 32638] avg=32768.0000 max=33022 max_rank=1 min=32603 min_rank=0 imbalance_ratio=1.007751 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=4 tokens_per_ep_rank=[32943, 32854, 32704, 32700, 32784, 32845, 32505, 32809] avg=32768.0000 max=32943 max_rank=0 min=32505 min_rank=6 imbalance_ratio=1.005341 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=4 tokens_per_ep_rank=[32943, 32854, 32704, 32700, 32784, 32845, 32505, 32809] avg=32768.0000 max=32943 max_rank=0 min=32505 min_rank=6 imbalance_ratio=1.005341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=4 tokens_per_ep_rank=[32943, 32854, 32704, 32700, 32784, 32845, 32505, 32809] avg=32768.0000 max=32943 max_rank=0 min=32505 min_rank=6 imbalance_ratio=1.005341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=4 tokens_per_ep_rank=[32943, 32854, 32704, 32700, 32784, 32845, 32505, 32809] avg=32768.0000 max=32943 max_rank=0 min=32505 min_rank=6 imbalance_ratio=1.005341 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=3 tokens_per_ep_rank=[32603, 33022, 32872, 32770, 32851, 32710, 32678, 32638] avg=32768.0000 max=33022 max_rank=1 min=32603 min_rank=0 imbalance_ratio=1.007751 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=4 tokens_per_ep_rank=[32943, 32854, 32704, 32700, 32784, 32845, 32505, 32809] avg=32768.0000 max=32943 max_rank=0 min=32505 min_rank=6 imbalance_ratio=1.005341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=4 tokens_per_ep_rank=[32943, 32854, 32704, 32700, 32784, 32845, 32505, 32809] avg=32768.0000 max=32943 max_rank=0 min=32505 min_rank=6 imbalance_ratio=1.005341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=4 tokens_per_ep_rank=[32943, 32854, 32704, 32700, 32784, 32845, 32505, 32809] avg=32768.0000 max=32943 max_rank=0 min=32505 min_rank=6 imbalance_ratio=1.005341 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=4 tokens_per_ep_rank=[32943, 32854, 32704, 32700, 32784, 32845, 32505, 32809] avg=32768.0000 max=32943 max_rank=0 min=32505 min_rank=6 imbalance_ratio=1.005341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=4 tokens_per_ep_rank=[32943, 32854, 32704, 32700, 32784, 32845, 32505, 32809] avg=32768.0000 max=32943 max_rank=0 min=32505 min_rank=6 imbalance_ratio=1.005341 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=5 tokens_per_ep_rank=[32710, 32846, 32787, 32620, 32929, 32888, 32594, 32770] avg=32768.0000 max=32929 max_rank=4 min=32594 min_rank=6 imbalance_ratio=1.004913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=4 tokens_per_ep_rank=[32943, 32854, 32704, 32700, 32784, 32845, 32505, 32809] avg=32768.0000 max=32943 max_rank=0 min=32505 min_rank=6 imbalance_ratio=1.005341 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=4 tokens_per_ep_rank=[32943, 32854, 32704, 32700, 32784, 32845, 32505, 32809] avg=32768.0000 max=32943 max_rank=0 min=32505 min_rank=6 imbalance_ratio=1.005341 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=5 tokens_per_ep_rank=[32710, 32846, 32787, 32620, 32929, 32888, 32594, 32770] avg=32768.0000 max=32929 max_rank=4 min=32594 min_rank=6 imbalance_ratio=1.004913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=4 tokens_per_ep_rank=[32943, 32854, 32704, 32700, 32784, 32845, 32505, 32809] avg=32768.0000 max=32943 max_rank=0 min=32505 min_rank=6 imbalance_ratio=1.005341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=4 tokens_per_ep_rank=[32943, 32854, 32704, 32700, 32784, 32845, 32505, 32809] avg=32768.0000 max=32943 max_rank=0 min=32505 min_rank=6 imbalance_ratio=1.005341 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=5 tokens_per_ep_rank=[32710, 32846, 32787, 32620, 32929, 32888, 32594, 32770] avg=32768.0000 max=32929 max_rank=4 min=32594 min_rank=6 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=5 tokens_per_ep_rank=[32710, 32846, 32787, 32620, 32929, 32888, 32594, 32770] avg=32768.0000 max=32929 max_rank=4 min=32594 min_rank=6 imbalance_ratio=1.004913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=5 tokens_per_ep_rank=[32710, 32846, 32787, 32620, 32929, 32888, 32594, 32770] avg=32768.0000 max=32929 max_rank=4 min=32594 min_rank=6 imbalance_ratio=1.004913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=5 tokens_per_ep_rank=[32710, 32846, 32787, 32620, 32929, 32888, 32594, 32770] avg=32768.0000 max=32929 max_rank=4 min=32594 min_rank=6 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=4 tokens_per_ep_rank=[32943, 32854, 32704, 32700, 32784, 32845, 32505, 32809] avg=32768.0000 max=32943 max_rank=0 min=32505 min_rank=6 imbalance_ratio=1.005341 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=5 tokens_per_ep_rank=[32710, 32846, 32787, 32620, 32929, 32888, 32594, 32770] avg=32768.0000 max=32929 max_rank=4 min=32594 min_rank=6 imbalance_ratio=1.004913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=5 tokens_per_ep_rank=[32710, 32846, 32787, 32620, 32929, 32888, 32594, 32770] avg=32768.0000 max=32929 max_rank=4 min=32594 min_rank=6 imbalance_ratio=1.004913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=5 tokens_per_ep_rank=[32710, 32846, 32787, 32620, 32929, 32888, 32594, 32770] avg=32768.0000 max=32929 max_rank=4 min=32594 min_rank=6 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=5 tokens_per_ep_rank=[32710, 32846, 32787, 32620, 32929, 32888, 32594, 32770] avg=32768.0000 max=32929 max_rank=4 min=32594 min_rank=6 imbalance_ratio=1.004913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=6 tokens_per_ep_rank=[32631, 32920, 32810, 32290, 33041, 32843, 32787, 32822] avg=32768.0000 max=33041 max_rank=4 min=32290 min_rank=3 imbalance_ratio=1.008331 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=5 tokens_per_ep_rank=[32710, 32846, 32787, 32620, 32929, 32888, 32594, 32770] avg=32768.0000 max=32929 max_rank=4 min=32594 min_rank=6 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=5 tokens_per_ep_rank=[32710, 32846, 32787, 32620, 32929, 32888, 32594, 32770] avg=32768.0000 max=32929 max_rank=4 min=32594 min_rank=6 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=5 tokens_per_ep_rank=[32710, 32846, 32787, 32620, 32929, 32888, 32594, 32770] avg=32768.0000 max=32929 max_rank=4 min=32594 min_rank=6 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=6 tokens_per_ep_rank=[32631, 32920, 32810, 32290, 33041, 32843, 32787, 32822] avg=32768.0000 max=33041 max_rank=4 min=32290 min_rank=3 imbalance_ratio=1.008331 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=5 tokens_per_ep_rank=[32710, 32846, 32787, 32620, 32929, 32888, 32594, 32770] avg=32768.0000 max=32929 max_rank=4 min=32594 min_rank=6 imbalance_ratio=1.004913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=5 tokens_per_ep_rank=[32710, 32846, 32787, 32620, 32929, 32888, 32594, 32770] avg=32768.0000 max=32929 max_rank=4 min=32594 min_rank=6 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=6 tokens_per_ep_rank=[32631, 32920, 32810, 32290, 33041, 32843, 32787, 32822] avg=32768.0000 max=33041 max_rank=4 min=32290 min_rank=3 imbalance_ratio=1.008331 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=6 tokens_per_ep_rank=[32631, 32920, 32810, 32290, 33041, 32843, 32787, 32822] avg=32768.0000 max=33041 max_rank=4 min=32290 min_rank=3 imbalance_ratio=1.008331 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=6 tokens_per_ep_rank=[32631, 32920, 32810, 32290, 33041, 32843, 32787, 32822] avg=32768.0000 max=33041 max_rank=4 min=32290 min_rank=3 imbalance_ratio=1.008331 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=6 tokens_per_ep_rank=[32631, 32920, 32810, 32290, 33041, 32843, 32787, 32822] avg=32768.0000 max=33041 max_rank=4 min=32290 min_rank=3 imbalance_ratio=1.008331 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=5 tokens_per_ep_rank=[32710, 32846, 32787, 32620, 32929, 32888, 32594, 32770] avg=32768.0000 max=32929 max_rank=4 min=32594 min_rank=6 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=6 tokens_per_ep_rank=[32631, 32920, 32810, 32290, 33041, 32843, 32787, 32822] avg=32768.0000 max=33041 max_rank=4 min=32290 min_rank=3 imbalance_ratio=1.008331 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=6 tokens_per_ep_rank=[32631, 32920, 32810, 32290, 33041, 32843, 32787, 32822] avg=32768.0000 max=33041 max_rank=4 min=32290 min_rank=3 imbalance_ratio=1.008331 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=6 tokens_per_ep_rank=[32631, 32920, 32810, 32290, 33041, 32843, 32787, 32822] avg=32768.0000 max=33041 max_rank=4 min=32290 min_rank=3 imbalance_ratio=1.008331 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=6 tokens_per_ep_rank=[32631, 32920, 32810, 32290, 33041, 32843, 32787, 32822] avg=32768.0000 max=33041 max_rank=4 min=32290 min_rank=3 imbalance_ratio=1.008331 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=7 tokens_per_ep_rank=[32889, 32708, 33070, 32588, 32656, 32726, 32782, 32725] avg=32768.0000 max=33070 max_rank=2 min=32588 min_rank=3 imbalance_ratio=1.009216 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=6 tokens_per_ep_rank=[32631, 32920, 32810, 32290, 33041, 32843, 32787, 32822] avg=32768.0000 max=33041 max_rank=4 min=32290 min_rank=3 imbalance_ratio=1.008331 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=6 tokens_per_ep_rank=[32631, 32920, 32810, 32290, 33041, 32843, 32787, 32822] avg=32768.0000 max=33041 max_rank=4 min=32290 min_rank=3 imbalance_ratio=1.008331 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=6 tokens_per_ep_rank=[32631, 32920, 32810, 32290, 33041, 32843, 32787, 32822] avg=32768.0000 max=33041 max_rank=4 min=32290 min_rank=3 imbalance_ratio=1.008331 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=6 tokens_per_ep_rank=[32631, 32920, 32810, 32290, 33041, 32843, 32787, 32822] avg=32768.0000 max=33041 max_rank=4 min=32290 min_rank=3 imbalance_ratio=1.008331 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=7 tokens_per_ep_rank=[32889, 32708, 33070, 32588, 32656, 32726, 32782, 32725] avg=32768.0000 max=33070 max_rank=2 min=32588 min_rank=3 imbalance_ratio=1.009216 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=6 tokens_per_ep_rank=[32631, 32920, 32810, 32290, 33041, 32843, 32787, 32822] avg=32768.0000 max=33041 max_rank=4 min=32290 min_rank=3 imbalance_ratio=1.008331 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=7 tokens_per_ep_rank=[32889, 32708, 33070, 32588, 32656, 32726, 32782, 32725] avg=32768.0000 max=33070 max_rank=2 min=32588 min_rank=3 imbalance_ratio=1.009216 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=7 tokens_per_ep_rank=[32889, 32708, 33070, 32588, 32656, 32726, 32782, 32725] avg=32768.0000 max=33070 max_rank=2 min=32588 min_rank=3 imbalance_ratio=1.009216 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=7 tokens_per_ep_rank=[32889, 32708, 33070, 32588, 32656, 32726, 32782, 32725] avg=32768.0000 max=33070 max_rank=2 min=32588 min_rank=3 imbalance_ratio=1.009216 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=7 tokens_per_ep_rank=[32889, 32708, 33070, 32588, 32656, 32726, 32782, 32725] avg=32768.0000 max=33070 max_rank=2 min=32588 min_rank=3 imbalance_ratio=1.009216 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=6 tokens_per_ep_rank=[32631, 32920, 32810, 32290, 33041, 32843, 32787, 32822] avg=32768.0000 max=33041 max_rank=4 min=32290 min_rank=3 imbalance_ratio=1.008331 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=7 tokens_per_ep_rank=[32889, 32708, 33070, 32588, 32656, 32726, 32782, 32725] avg=32768.0000 max=33070 max_rank=2 min=32588 min_rank=3 imbalance_ratio=1.009216 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=7 tokens_per_ep_rank=[32889, 32708, 33070, 32588, 32656, 32726, 32782, 32725] avg=32768.0000 max=33070 max_rank=2 min=32588 min_rank=3 imbalance_ratio=1.009216 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=7 tokens_per_ep_rank=[32889, 32708, 33070, 32588, 32656, 32726, 32782, 32725] avg=32768.0000 max=33070 max_rank=2 min=32588 min_rank=3 imbalance_ratio=1.009216 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=7 tokens_per_ep_rank=[32889, 32708, 33070, 32588, 32656, 32726, 32782, 32725] avg=32768.0000 max=33070 max_rank=2 min=32588 min_rank=3 imbalance_ratio=1.009216 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=8 tokens_per_ep_rank=[32606, 32773, 32885, 32820, 32616, 32836, 32857, 32751] avg=32768.0000 max=32885 max_rank=2 min=32606 min_rank=0 imbalance_ratio=1.003571 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=7 tokens_per_ep_rank=[32889, 32708, 33070, 32588, 32656, 32726, 32782, 32725] avg=32768.0000 max=33070 max_rank=2 min=32588 min_rank=3 imbalance_ratio=1.009216 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=7 tokens_per_ep_rank=[32889, 32708, 33070, 32588, 32656, 32726, 32782, 32725] avg=32768.0000 max=33070 max_rank=2 min=32588 min_rank=3 imbalance_ratio=1.009216 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=7 tokens_per_ep_rank=[32889, 32708, 33070, 32588, 32656, 32726, 32782, 32725] avg=32768.0000 max=33070 max_rank=2 min=32588 min_rank=3 imbalance_ratio=1.009216 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=7 tokens_per_ep_rank=[32889, 32708, 33070, 32588, 32656, 32726, 32782, 32725] avg=32768.0000 max=33070 max_rank=2 min=32588 min_rank=3 imbalance_ratio=1.009216 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=8 tokens_per_ep_rank=[32606, 32773, 32885, 32820, 32616, 32836, 32857, 32751] avg=32768.0000 max=32885 max_rank=2 min=32606 min_rank=0 imbalance_ratio=1.003571 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=7 tokens_per_ep_rank=[32889, 32708, 33070, 32588, 32656, 32726, 32782, 32725] avg=32768.0000 max=33070 max_rank=2 min=32588 min_rank=3 imbalance_ratio=1.009216 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=8 tokens_per_ep_rank=[32606, 32773, 32885, 32820, 32616, 32836, 32857, 32751] avg=32768.0000 max=32885 max_rank=2 min=32606 min_rank=0 imbalance_ratio=1.003571 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=8 tokens_per_ep_rank=[32606, 32773, 32885, 32820, 32616, 32836, 32857, 32751] avg=32768.0000 max=32885 max_rank=2 min=32606 min_rank=0 imbalance_ratio=1.003571 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=8 tokens_per_ep_rank=[32606, 32773, 32885, 32820, 32616, 32836, 32857, 32751] avg=32768.0000 max=32885 max_rank=2 min=32606 min_rank=0 imbalance_ratio=1.003571 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=8 tokens_per_ep_rank=[32606, 32773, 32885, 32820, 32616, 32836, 32857, 32751] avg=32768.0000 max=32885 max_rank=2 min=32606 min_rank=0 imbalance_ratio=1.003571 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=7 tokens_per_ep_rank=[32889, 32708, 33070, 32588, 32656, 32726, 32782, 32725] avg=32768.0000 max=33070 max_rank=2 min=32588 min_rank=3 imbalance_ratio=1.009216 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=8 tokens_per_ep_rank=[32606, 32773, 32885, 32820, 32616, 32836, 32857, 32751] avg=32768.0000 max=32885 max_rank=2 min=32606 min_rank=0 imbalance_ratio=1.003571 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=8 tokens_per_ep_rank=[32606, 32773, 32885, 32820, 32616, 32836, 32857, 32751] avg=32768.0000 max=32885 max_rank=2 min=32606 min_rank=0 imbalance_ratio=1.003571 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=8 tokens_per_ep_rank=[32606, 32773, 32885, 32820, 32616, 32836, 32857, 32751] avg=32768.0000 max=32885 max_rank=2 min=32606 min_rank=0 imbalance_ratio=1.003571 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=9 tokens_per_ep_rank=[32791, 32682, 32540, 33183, 32803, 32808, 32769, 32568] avg=32768.0000 max=33183 max_rank=3 min=32540 min_rank=2 imbalance_ratio=1.012665 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=8 tokens_per_ep_rank=[32606, 32773, 32885, 32820, 32616, 32836, 32857, 32751] avg=32768.0000 max=32885 max_rank=2 min=32606 min_rank=0 imbalance_ratio=1.003571 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=8 tokens_per_ep_rank=[32606, 32773, 32885, 32820, 32616, 32836, 32857, 32751] avg=32768.0000 max=32885 max_rank=2 min=32606 min_rank=0 imbalance_ratio=1.003571 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=8 tokens_per_ep_rank=[32606, 32773, 32885, 32820, 32616, 32836, 32857, 32751] avg=32768.0000 max=32885 max_rank=2 min=32606 min_rank=0 imbalance_ratio=1.003571 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=8 tokens_per_ep_rank=[32606, 32773, 32885, 32820, 32616, 32836, 32857, 32751] avg=32768.0000 max=32885 max_rank=2 min=32606 min_rank=0 imbalance_ratio=1.003571 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=9 tokens_per_ep_rank=[32791, 32682, 32627, 33096, 32803, 32808, 32769, 32568] avg=32768.0000 max=33096 max_rank=3 min=32568 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=87 max_load_reduction_pct=0.002622 moved_to_replicas=87 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=9 tokens_per_ep_rank=[32791, 32682, 32540, 33183, 32803, 32808, 32769, 32568] avg=32768.0000 max=33183 max_rank=3 min=32540 min_rank=2 imbalance_ratio=1.012665 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=8 tokens_per_ep_rank=[32606, 32773, 32885, 32820, 32616, 32836, 32857, 32751] avg=32768.0000 max=32885 max_rank=2 min=32606 min_rank=0 imbalance_ratio=1.003571 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=8 tokens_per_ep_rank=[32606, 32773, 32885, 32820, 32616, 32836, 32857, 32751] avg=32768.0000 max=32885 max_rank=2 min=32606 min_rank=0 imbalance_ratio=1.003571 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=9 tokens_per_ep_rank=[32791, 32682, 32540, 33183, 32803, 32808, 32769, 32568] avg=32768.0000 max=33183 max_rank=3 min=32540 min_rank=2 imbalance_ratio=1.012665 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=9 tokens_per_ep_rank=[32791, 32682, 32540, 33183, 32803, 32808, 32769, 32568] avg=32768.0000 max=33183 max_rank=3 min=32540 min_rank=2 imbalance_ratio=1.012665 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=9 tokens_per_ep_rank=[32791, 32682, 32540, 33183, 32803, 32808, 32769, 32568] avg=32768.0000 max=33183 max_rank=3 min=32540 min_rank=2 imbalance_ratio=1.012665 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=9 tokens_per_ep_rank=[32791, 32682, 32627, 33096, 32803, 32808, 32769, 32568] avg=32768.0000 max=33096 max_rank=3 min=32568 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=87 max_load_reduction_pct=0.002622 moved_to_replicas=87 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=8 tokens_per_ep_rank=[32606, 32773, 32885, 32820, 32616, 32836, 32857, 32751] avg=32768.0000 max=32885 max_rank=2 min=32606 min_rank=0 imbalance_ratio=1.003571 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=9 tokens_per_ep_rank=[32791, 32682, 32627, 33096, 32803, 32808, 32769, 32568] avg=32768.0000 max=33096 max_rank=3 min=32568 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=87 max_load_reduction_pct=0.002622 moved_to_replicas=87 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=9 tokens_per_ep_rank=[32791, 32682, 32540, 33183, 32803, 32808, 32769, 32568] avg=32768.0000 max=33183 max_rank=3 min=32540 min_rank=2 imbalance_ratio=1.012665 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=10 tokens_per_ep_rank=[32753, 32782, 32806, 32728, 32939, 32824, 32774, 32538] avg=32768.0000 max=32939 max_rank=4 min=32538 min_rank=7 imbalance_ratio=1.005219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=9 tokens_per_ep_rank=[32791, 32682, 32540, 33183, 32803, 32808, 32769, 32568] avg=32768.0000 max=33183 max_rank=3 min=32540 min_rank=2 imbalance_ratio=1.012665 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=9 tokens_per_ep_rank=[32791, 32682, 32627, 33096, 32803, 32808, 32769, 32568] avg=32768.0000 max=33096 max_rank=3 min=32568 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=87 max_load_reduction_pct=0.002622 moved_to_replicas=87 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=9 tokens_per_ep_rank=[32791, 32682, 32627, 33096, 32803, 32808, 32769, 32568] avg=32768.0000 max=33096 max_rank=3 min=32568 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=87 max_load_reduction_pct=0.002622 moved_to_replicas=87 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=9 tokens_per_ep_rank=[32791, 32682, 32627, 33096, 32803, 32808, 32769, 32568] avg=32768.0000 max=33096 max_rank=3 min=32568 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=87 max_load_reduction_pct=0.002622 moved_to_replicas=87 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=10 tokens_per_ep_rank=[32753, 32782, 32806, 32728, 32939, 32824, 32774, 32538] avg=32768.0000 max=32939 max_rank=4 min=32538 min_rank=7 imbalance_ratio=1.005219 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=10 tokens_per_ep_rank=[32753, 32782, 32806, 32728, 32939, 32824, 32774, 32538] avg=32768.0000 max=32939 max_rank=4 min=32538 min_rank=7 imbalance_ratio=1.005219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=9 tokens_per_ep_rank=[32791, 32682, 32540, 33183, 32803, 32808, 32769, 32568] avg=32768.0000 max=33183 max_rank=3 min=32540 min_rank=2 imbalance_ratio=1.012665 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=10 tokens_per_ep_rank=[32753, 32782, 32806, 32728, 32939, 32824, 32774, 32538] avg=32768.0000 max=32939 max_rank=4 min=32538 min_rank=7 imbalance_ratio=1.005219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=9 tokens_per_ep_rank=[32791, 32682, 32627, 33096, 32803, 32808, 32769, 32568] avg=32768.0000 max=33096 max_rank=3 min=32568 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=87 max_load_reduction_pct=0.002622 moved_to_replicas=87 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=10 tokens_per_ep_rank=[32753, 32782, 32806, 32728, 32939, 32824, 32774, 32538] avg=32768.0000 max=32939 max_rank=4 min=32538 min_rank=7 imbalance_ratio=1.005219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=10 tokens_per_ep_rank=[32753, 32782, 32806, 32728, 32939, 32824, 32774, 32538] avg=32768.0000 max=32939 max_rank=4 min=32538 min_rank=7 imbalance_ratio=1.005219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=10 tokens_per_ep_rank=[32753, 32782, 32806, 32728, 32939, 32824, 32774, 32538] avg=32768.0000 max=32939 max_rank=4 min=32538 min_rank=7 imbalance_ratio=1.005219 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=9 tokens_per_ep_rank=[32791, 32682, 32627, 33096, 32803, 32808, 32769, 32568] avg=32768.0000 max=33096 max_rank=3 min=32568 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=87 max_load_reduction_pct=0.002622 moved_to_replicas=87 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=10 tokens_per_ep_rank=[32753, 32782, 32806, 32728, 32939, 32824, 32774, 32538] avg=32768.0000 max=32939 max_rank=4 min=32538 min_rank=7 imbalance_ratio=1.005219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=10 tokens_per_ep_rank=[32753, 32782, 32806, 32728, 32939, 32824, 32774, 32538] avg=32768.0000 max=32939 max_rank=4 min=32538 min_rank=7 imbalance_ratio=1.005219 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=10 tokens_per_ep_rank=[32753, 32782, 32806, 32728, 32939, 32824, 32774, 32538] avg=32768.0000 max=32939 max_rank=4 min=32538 min_rank=7 imbalance_ratio=1.005219 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=10 tokens_per_ep_rank=[32753, 32782, 32806, 32728, 32939, 32824, 32774, 32538] avg=32768.0000 max=32939 max_rank=4 min=32538 min_rank=7 imbalance_ratio=1.005219 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=10 tokens_per_ep_rank=[32753, 32782, 32806, 32728, 32939, 32824, 32774, 32538] avg=32768.0000 max=32939 max_rank=4 min=32538 min_rank=7 imbalance_ratio=1.005219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=10 tokens_per_ep_rank=[32753, 32782, 32806, 32728, 32939, 32824, 32774, 32538] avg=32768.0000 max=32939 max_rank=4 min=32538 min_rank=7 imbalance_ratio=1.005219 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=10 tokens_per_ep_rank=[32753, 32782, 32806, 32728, 32939, 32824, 32774, 32538] avg=32768.0000 max=32939 max_rank=4 min=32538 min_rank=7 imbalance_ratio=1.005219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=10 tokens_per_ep_rank=[32753, 32782, 32806, 32728, 32939, 32824, 32774, 32538] avg=32768.0000 max=32939 max_rank=4 min=32538 min_rank=7 imbalance_ratio=1.005219 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=10 tokens_per_ep_rank=[32753, 32782, 32806, 32728, 32939, 32824, 32774, 32538] avg=32768.0000 max=32939 max_rank=4 min=32538 min_rank=7 imbalance_ratio=1.005219 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=9 records=10 total_expected_recv=327605 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76544 peak_demand=32888 required_capacity=1.003662 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=9 records=10 total_expected_recv=328381 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=75776 peak_demand=33070 required_capacity=1.009216 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=9 records=10 total_expected_recv=327391 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76544 peak_demand=32999 required_capacity=1.007050 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=9 records=10 total_expected_recv=327331 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=77184 peak_demand=32857 required_capacity=1.002716 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=9 records=10 total_expected_recv=327993 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76032 peak_demand=33041 required_capacity=1.008331 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=9 records=10 total_expected_recv=328146 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76288 peak_demand=33022 required_capacity=1.007751 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=9 records=10 total_expected_recv=327358 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76800 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=9 records=10 total_expected_recv=327235 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=77312 peak_demand=32943 required_capacity=1.005341 + [2026-08-26 15:53:13] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 212.0 | throughput per GPU (TFLOP/s/GPU): 582.4 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.547990E+00 | loss scale: 1.0 | grad norm: 3.710 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 33664 | min safe capacity: 1.0273 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32826 expected_overflow=0 exact_recv_per_rank=[32863, 32801, 32734, 32630, 32826, 32823, 32659, 32808] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32808 expected_overflow=0 exact_recv_per_rank=[32863, 32801, 32734, 32630, 32826, 32823, 32659, 32808] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32823 expected_overflow=0 exact_recv_per_rank=[32863, 32801, 32734, 32630, 32826, 32823, 32659, 32808] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32659 expected_overflow=0 exact_recv_per_rank=[32863, 32801, 32734, 32630, 32826, 32823, 32659, 32808] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32734 expected_overflow=0 exact_recv_per_rank=[32863, 32801, 32734, 32630, 32826, 32823, 32659, 32808] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32801 expected_overflow=0 exact_recv_per_rank=[32863, 32801, 32734, 32630, 32826, 32823, 32659, 32808] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32630 expected_overflow=0 exact_recv_per_rank=[32863, 32801, 32734, 32630, 32826, 32823, 32659, 32808] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32826 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32863 expected_overflow=0 exact_recv_per_rank=[32863, 32801, 32734, 32630, 32826, 32823, 32659, 32808] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32808 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32823 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32659 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32734 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32801 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32630 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32863 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=1 calls=1 budget=40960 expected_recv=32867 expected_overflow=0 exact_recv_per_rank=[32575, 32917, 32775, 32667, 32867, 32616, 32845, 32882] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=1 calls=1 budget=40960 expected_recv=32616 expected_overflow=0 exact_recv_per_rank=[32575, 32917, 32775, 32667, 32867, 32616, 32845, 32882] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=1 calls=1 budget=40960 expected_recv=32775 expected_overflow=0 exact_recv_per_rank=[32575, 32917, 32775, 32667, 32867, 32616, 32845, 32882] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=1 calls=1 budget=40960 expected_recv=32882 expected_overflow=0 exact_recv_per_rank=[32575, 32917, 32775, 32667, 32867, 32616, 32845, 32882] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=1 calls=1 budget=40960 expected_recv=32845 expected_overflow=0 exact_recv_per_rank=[32575, 32917, 32775, 32667, 32867, 32616, 32845, 32882] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=1 calls=1 budget=40960 expected_recv=32867 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=1 calls=1 budget=40960 expected_recv=32616 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=1 calls=1 budget=40960 expected_recv=32775 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=1 calls=1 budget=40960 expected_recv=32845 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=1 calls=1 budget=40960 expected_recv=32882 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=1 calls=1 budget=40960 expected_recv=32917 expected_overflow=0 exact_recv_per_rank=[32575, 32917, 32775, 32667, 32867, 32616, 32845, 32882] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=1 calls=1 budget=40960 expected_recv=32667 expected_overflow=0 exact_recv_per_rank=[32575, 32917, 32775, 32667, 32867, 32616, 32845, 32882] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=1 calls=1 budget=40960 expected_recv=32575 expected_overflow=0 exact_recv_per_rank=[32575, 32917, 32775, 32667, 32867, 32616, 32845, 32882] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=1 calls=1 budget=40960 expected_recv=32917 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=1 calls=1 budget=40960 expected_recv=32667 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=1 calls=1 budget=40960 expected_recv=32575 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=2 calls=1 budget=40960 expected_recv=32911 expected_overflow=0 exact_recv_per_rank=[32392, 32752, 32540, 32818, 32911, 32894, 32883, 32954] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=2 calls=1 budget=40960 expected_recv=32894 expected_overflow=0 exact_recv_per_rank=[32392, 32752, 32540, 32818, 32911, 32894, 32883, 32954] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=2 calls=1 budget=40960 expected_recv=32540 expected_overflow=0 exact_recv_per_rank=[32392, 32752, 32540, 32818, 32911, 32894, 32883, 32954] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=2 calls=1 budget=40960 expected_recv=32911 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=2 calls=1 budget=40960 expected_recv=32894 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=2 calls=1 budget=40960 expected_recv=32540 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=2 calls=1 budget=40960 expected_recv=32883 expected_overflow=0 exact_recv_per_rank=[32392, 32752, 32540, 32818, 32911, 32894, 32883, 32954] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=2 calls=1 budget=40960 expected_recv=32954 expected_overflow=0 exact_recv_per_rank=[32392, 32752, 32540, 32818, 32911, 32894, 32883, 32954] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=2 calls=1 budget=40960 expected_recv=32954 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=2 calls=1 budget=40960 expected_recv=32883 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=2 calls=1 budget=40960 expected_recv=32752 expected_overflow=0 exact_recv_per_rank=[32392, 32752, 32540, 32818, 32911, 32894, 32883, 32954] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=2 calls=1 budget=40960 expected_recv=32818 expected_overflow=0 exact_recv_per_rank=[32392, 32752, 32540, 32818, 32911, 32894, 32883, 32954] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=2 calls=1 budget=40960 expected_recv=32392 expected_overflow=0 exact_recv_per_rank=[32392, 32752, 32540, 32818, 32911, 32894, 32883, 32954] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=2 calls=1 budget=40960 expected_recv=32752 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=2 calls=1 budget=40960 expected_recv=32818 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=2 calls=1 budget=40960 expected_recv=32392 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=3 calls=1 budget=40960 expected_recv=32735 expected_overflow=0 exact_recv_per_rank=[32847, 32776, 33012, 32709, 32652, 32735, 32816, 32597] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=3 calls=1 budget=40960 expected_recv=32652 expected_overflow=0 exact_recv_per_rank=[32847, 32776, 33012, 32709, 32652, 32735, 32816, 32597] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=3 calls=1 budget=40960 expected_recv=32735 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=3 calls=1 budget=40960 expected_recv=33012 expected_overflow=0 exact_recv_per_rank=[32847, 32776, 33012, 32709, 32652, 32735, 32816, 32597] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=3 calls=1 budget=40960 expected_recv=32652 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=3 calls=1 budget=40960 expected_recv=32597 expected_overflow=0 exact_recv_per_rank=[32847, 32776, 33012, 32709, 32652, 32735, 32816, 32597] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=3 calls=1 budget=40960 expected_recv=33012 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=3 calls=1 budget=40960 expected_recv=32816 expected_overflow=0 exact_recv_per_rank=[32847, 32776, 33012, 32709, 32652, 32735, 32816, 32597] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=3 calls=1 budget=40960 expected_recv=32776 expected_overflow=0 exact_recv_per_rank=[32847, 32776, 33012, 32709, 32652, 32735, 32816, 32597] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=3 calls=1 budget=40960 expected_recv=32597 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=3 calls=1 budget=40960 expected_recv=32709 expected_overflow=0 exact_recv_per_rank=[32847, 32776, 33012, 32709, 32652, 32735, 32816, 32597] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=3 calls=1 budget=40960 expected_recv=32816 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=3 calls=1 budget=40960 expected_recv=32776 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=3 calls=1 budget=40960 expected_recv=32847 expected_overflow=0 exact_recv_per_rank=[32847, 32776, 33012, 32709, 32652, 32735, 32816, 32597] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=3 calls=1 budget=40960 expected_recv=32709 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=3 calls=1 budget=40960 expected_recv=32847 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=4 calls=1 budget=40960 expected_recv=32698 expected_overflow=0 exact_recv_per_rank=[32609, 32766, 32558, 32924, 32757, 32698, 32929, 32903] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=4 calls=1 budget=40960 expected_recv=32757 expected_overflow=0 exact_recv_per_rank=[32609, 32766, 32558, 32924, 32757, 32698, 32929, 32903] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=4 calls=1 budget=40960 expected_recv=32698 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=4 calls=1 budget=40960 expected_recv=32558 expected_overflow=0 exact_recv_per_rank=[32609, 32766, 32558, 32924, 32757, 32698, 32929, 32903] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=4 calls=1 budget=40960 expected_recv=32757 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=4 calls=1 budget=40960 expected_recv=32903 expected_overflow=0 exact_recv_per_rank=[32609, 32766, 32558, 32924, 32757, 32698, 32929, 32903] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=4 calls=1 budget=40960 expected_recv=32558 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=4 calls=1 budget=40960 expected_recv=32929 expected_overflow=0 exact_recv_per_rank=[32609, 32766, 32558, 32924, 32757, 32698, 32929, 32903] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=4 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32609, 32766, 32558, 32924, 32757, 32698, 32929, 32903] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=4 calls=1 budget=40960 expected_recv=32903 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=4 calls=1 budget=40960 expected_recv=32924 expected_overflow=0 exact_recv_per_rank=[32609, 32766, 32558, 32924, 32757, 32698, 32929, 32903] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=4 calls=1 budget=40960 expected_recv=32929 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=4 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=4 calls=1 budget=40960 expected_recv=32924 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=4 calls=1 budget=40960 expected_recv=32609 expected_overflow=0 exact_recv_per_rank=[32609, 32766, 32558, 32924, 32757, 32698, 32929, 32903] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=5 calls=1 budget=40960 expected_recv=32796 expected_overflow=0 exact_recv_per_rank=[32711, 32763, 32591, 32901, 32862, 32796, 32654, 32866] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=4 calls=1 budget=40960 expected_recv=32609 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=5 calls=1 budget=40960 expected_recv=32862 expected_overflow=0 exact_recv_per_rank=[32711, 32763, 32591, 32901, 32862, 32796, 32654, 32866] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=5 calls=1 budget=40960 expected_recv=32796 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=5 calls=1 budget=40960 expected_recv=32591 expected_overflow=0 exact_recv_per_rank=[32711, 32763, 32591, 32901, 32862, 32796, 32654, 32866] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=5 calls=1 budget=40960 expected_recv=32862 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=5 calls=1 budget=40960 expected_recv=32866 expected_overflow=0 exact_recv_per_rank=[32711, 32763, 32591, 32901, 32862, 32796, 32654, 32866] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=5 calls=1 budget=40960 expected_recv=32591 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=5 calls=1 budget=40960 expected_recv=32654 expected_overflow=0 exact_recv_per_rank=[32711, 32763, 32591, 32901, 32862, 32796, 32654, 32866] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=5 calls=1 budget=40960 expected_recv=32763 expected_overflow=0 exact_recv_per_rank=[32711, 32763, 32591, 32901, 32862, 32796, 32654, 32866] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=5 calls=1 budget=40960 expected_recv=32866 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=5 calls=1 budget=40960 expected_recv=32901 expected_overflow=0 exact_recv_per_rank=[32711, 32763, 32591, 32901, 32862, 32796, 32654, 32866] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=5 calls=1 budget=40960 expected_recv=32654 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=5 calls=1 budget=40960 expected_recv=32763 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=5 calls=1 budget=40960 expected_recv=32901 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=5 calls=1 budget=40960 expected_recv=32711 expected_overflow=0 exact_recv_per_rank=[32711, 32763, 32591, 32901, 32862, 32796, 32654, 32866] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=6 calls=1 budget=40960 expected_recv=32607 expected_overflow=0 exact_recv_per_rank=[32813, 32699, 32594, 32780, 32762, 32607, 32967, 32922] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=5 calls=1 budget=40960 expected_recv=32711 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=6 calls=1 budget=40960 expected_recv=32607 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=6 calls=1 budget=40960 expected_recv=32762 expected_overflow=0 exact_recv_per_rank=[32813, 32699, 32594, 32780, 32762, 32607, 32967, 32922] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=6 calls=1 budget=40960 expected_recv=32594 expected_overflow=0 exact_recv_per_rank=[32813, 32699, 32594, 32780, 32762, 32607, 32967, 32922] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=6 calls=1 budget=40960 expected_recv=32762 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=6 calls=1 budget=40960 expected_recv=32594 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=6 calls=1 budget=40960 expected_recv=32922 expected_overflow=0 exact_recv_per_rank=[32813, 32699, 32594, 32780, 32762, 32607, 32967, 32922] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=6 calls=1 budget=40960 expected_recv=32967 expected_overflow=0 exact_recv_per_rank=[32813, 32699, 32594, 32780, 32762, 32607, 32967, 32922] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=6 calls=1 budget=40960 expected_recv=32699 expected_overflow=0 exact_recv_per_rank=[32813, 32699, 32594, 32780, 32762, 32607, 32967, 32922] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=6 calls=1 budget=40960 expected_recv=32922 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=6 calls=1 budget=40960 expected_recv=32780 expected_overflow=0 exact_recv_per_rank=[32813, 32699, 32594, 32780, 32762, 32607, 32967, 32922] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=6 calls=1 budget=40960 expected_recv=32967 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=6 calls=1 budget=40960 expected_recv=32699 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=6 calls=1 budget=40960 expected_recv=32780 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=6 calls=1 budget=40960 expected_recv=32813 expected_overflow=0 exact_recv_per_rank=[32813, 32699, 32594, 32780, 32762, 32607, 32967, 32922] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=7 calls=1 budget=40960 expected_recv=32638 expected_overflow=0 exact_recv_per_rank=[33053, 32578, 32736, 32828, 32899, 32638, 32509, 32903] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=7 calls=1 budget=40960 expected_recv=32899 expected_overflow=0 exact_recv_per_rank=[33053, 32578, 32736, 32828, 32899, 32638, 32509, 32903] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=6 calls=1 budget=40960 expected_recv=32813 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=7 calls=1 budget=40960 expected_recv=32638 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=7 calls=1 budget=40960 expected_recv=32736 expected_overflow=0 exact_recv_per_rank=[33053, 32578, 32736, 32828, 32899, 32638, 32509, 32903] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=7 calls=1 budget=40960 expected_recv=32899 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=7 calls=1 budget=40960 expected_recv=32736 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=7 calls=1 budget=40960 expected_recv=32903 expected_overflow=0 exact_recv_per_rank=[33053, 32578, 32736, 32828, 32899, 32638, 32509, 32903] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=7 calls=1 budget=40960 expected_recv=32509 expected_overflow=0 exact_recv_per_rank=[33053, 32578, 32736, 32828, 32899, 32638, 32509, 32903] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=7 calls=1 budget=40960 expected_recv=32578 expected_overflow=0 exact_recv_per_rank=[33053, 32578, 32736, 32828, 32899, 32638, 32509, 32903] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=7 calls=1 budget=40960 expected_recv=32828 expected_overflow=0 exact_recv_per_rank=[33053, 32578, 32736, 32828, 32899, 32638, 32509, 32903] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=7 calls=1 budget=40960 expected_recv=32903 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=7 calls=1 budget=40960 expected_recv=32509 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=7 calls=1 budget=40960 expected_recv=32578 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=7 calls=1 budget=40960 expected_recv=32828 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=7 calls=1 budget=40960 expected_recv=33053 expected_overflow=0 exact_recv_per_rank=[33053, 32578, 32736, 32828, 32899, 32638, 32509, 32903] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=8 calls=1 budget=40960 expected_recv=32870 expected_overflow=0 exact_recv_per_rank=[32678, 32880, 32830, 32764, 32963, 32870, 32425, 32734] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=7 calls=1 budget=40960 expected_recv=33053 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=8 calls=1 budget=40960 expected_recv=32963 expected_overflow=0 exact_recv_per_rank=[32678, 32880, 32830, 32764, 32963, 32870, 32425, 32734] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=8 calls=1 budget=40960 expected_recv=32870 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=8 calls=1 budget=40960 expected_recv=32830 expected_overflow=0 exact_recv_per_rank=[32678, 32880, 32830, 32764, 32963, 32870, 32425, 32734] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=8 calls=1 budget=40960 expected_recv=32963 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=8 calls=1 budget=40960 expected_recv=32830 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=8 calls=1 budget=40960 expected_recv=32734 expected_overflow=0 exact_recv_per_rank=[32678, 32880, 32830, 32764, 32963, 32870, 32425, 32734] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=8 calls=1 budget=40960 expected_recv=32425 expected_overflow=0 exact_recv_per_rank=[32678, 32880, 32830, 32764, 32963, 32870, 32425, 32734] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=8 calls=1 budget=40960 expected_recv=32880 expected_overflow=0 exact_recv_per_rank=[32678, 32880, 32830, 32764, 32963, 32870, 32425, 32734] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=8 calls=1 budget=40960 expected_recv=32764 expected_overflow=0 exact_recv_per_rank=[32678, 32880, 32830, 32764, 32963, 32870, 32425, 32734] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=8 calls=1 budget=40960 expected_recv=32734 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=8 calls=1 budget=40960 expected_recv=32425 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=8 calls=1 budget=40960 expected_recv=32880 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=8 calls=1 budget=40960 expected_recv=32764 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=8 calls=1 budget=40960 expected_recv=32678 expected_overflow=0 exact_recv_per_rank=[32678, 32880, 32830, 32764, 32963, 32870, 32425, 32734] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=9 calls=1 budget=40960 expected_recv=32841 expected_overflow=0 exact_recv_per_rank=[32562, 33011, 32494, 33010, 32749, 32841, 32925, 32552] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=9 calls=1 budget=40960 expected_recv=32749 expected_overflow=0 exact_recv_per_rank=[32562, 33011, 32494, 33010, 32749, 32841, 32925, 32552] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=9 calls=1 budget=40960 expected_recv=32494 expected_overflow=0 exact_recv_per_rank=[32562, 33011, 32494, 33010, 32749, 32841, 32925, 32552] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=8 calls=1 budget=40960 expected_recv=32678 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=9 calls=1 budget=40960 expected_recv=32841 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=9 calls=1 budget=40960 expected_recv=32749 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=9 calls=1 budget=40960 expected_recv=32494 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=9 calls=1 budget=40960 expected_recv=32552 expected_overflow=0 exact_recv_per_rank=[32562, 33011, 32494, 33010, 32749, 32841, 32925, 32552] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=9 calls=1 budget=40960 expected_recv=32925 expected_overflow=0 exact_recv_per_rank=[32562, 33011, 32494, 33010, 32749, 32841, 32925, 32552] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=9 calls=1 budget=40960 expected_recv=33011 expected_overflow=0 exact_recv_per_rank=[32562, 33011, 32494, 33010, 32749, 32841, 32925, 32552] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=9 calls=1 budget=40960 expected_recv=32552 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=9 calls=1 budget=40960 expected_recv=33010 expected_overflow=0 exact_recv_per_rank=[32562, 33011, 32494, 33010, 32749, 32841, 32925, 32552] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=9 calls=1 budget=40960 expected_recv=32925 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=9 calls=1 budget=40960 expected_recv=33011 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=9 calls=1 budget=40960 expected_recv=33010 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=9 calls=1 budget=40960 expected_recv=32562 expected_overflow=0 exact_recv_per_rank=[32562, 33011, 32494, 33010, 32749, 32841, 32925, 32552] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=9 calls=1 budget=40960 expected_recv=32562 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=1 tokens_per_ep_rank=[32863, 32801, 32734, 32630, 32826, 32823, 32659, 32808] avg=32768.0000 max=32863 max_rank=0 min=32630 min_rank=3 imbalance_ratio=1.002899 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=1 tokens_per_ep_rank=[32863, 32801, 32734, 32630, 32826, 32823, 32659, 32808] avg=32768.0000 max=32863 max_rank=0 min=32630 min_rank=3 imbalance_ratio=1.002899 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=1 tokens_per_ep_rank=[32863, 32801, 32734, 32630, 32826, 32823, 32659, 32808] avg=32768.0000 max=32863 max_rank=0 min=32630 min_rank=3 imbalance_ratio=1.002899 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=1 tokens_per_ep_rank=[32863, 32801, 32734, 32630, 32826, 32823, 32659, 32808] avg=32768.0000 max=32863 max_rank=0 min=32630 min_rank=3 imbalance_ratio=1.002899 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=1 tokens_per_ep_rank=[32863, 32801, 32734, 32630, 32826, 32823, 32659, 32808] avg=32768.0000 max=32863 max_rank=0 min=32630 min_rank=3 imbalance_ratio=1.002899 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=1 tokens_per_ep_rank=[32863, 32801, 32734, 32630, 32826, 32823, 32659, 32808] avg=32768.0000 max=32863 max_rank=0 min=32630 min_rank=3 imbalance_ratio=1.002899 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=1 tokens_per_ep_rank=[32863, 32801, 32734, 32630, 32826, 32823, 32659, 32808] avg=32768.0000 max=32863 max_rank=0 min=32630 min_rank=3 imbalance_ratio=1.002899 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=1 tokens_per_ep_rank=[32863, 32801, 32734, 32630, 32826, 32823, 32659, 32808] avg=32768.0000 max=32863 max_rank=0 min=32630 min_rank=3 imbalance_ratio=1.002899 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=1 tokens_per_ep_rank=[32863, 32801, 32734, 32630, 32826, 32823, 32659, 32808] avg=32768.0000 max=32863 max_rank=0 min=32630 min_rank=3 imbalance_ratio=1.002899 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=1 tokens_per_ep_rank=[32863, 32801, 32734, 32630, 32826, 32823, 32659, 32808] avg=32768.0000 max=32863 max_rank=0 min=32630 min_rank=3 imbalance_ratio=1.002899 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=1 tokens_per_ep_rank=[32863, 32801, 32734, 32630, 32826, 32823, 32659, 32808] avg=32768.0000 max=32863 max_rank=0 min=32630 min_rank=3 imbalance_ratio=1.002899 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=1 tokens_per_ep_rank=[32863, 32801, 32734, 32630, 32826, 32823, 32659, 32808] avg=32768.0000 max=32863 max_rank=0 min=32630 min_rank=3 imbalance_ratio=1.002899 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=1 tokens_per_ep_rank=[32863, 32801, 32734, 32630, 32826, 32823, 32659, 32808] avg=32768.0000 max=32863 max_rank=0 min=32630 min_rank=3 imbalance_ratio=1.002899 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=2 tokens_per_ep_rank=[32575, 32917, 32775, 32667, 32867, 32616, 32845, 32882] avg=32768.0000 max=32917 max_rank=1 min=32575 min_rank=0 imbalance_ratio=1.004547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=1 tokens_per_ep_rank=[32863, 32801, 32734, 32630, 32826, 32823, 32659, 32808] avg=32768.0000 max=32863 max_rank=0 min=32630 min_rank=3 imbalance_ratio=1.002899 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=1 tokens_per_ep_rank=[32863, 32801, 32734, 32630, 32826, 32823, 32659, 32808] avg=32768.0000 max=32863 max_rank=0 min=32630 min_rank=3 imbalance_ratio=1.002899 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=2 tokens_per_ep_rank=[32575, 32917, 32775, 32667, 32867, 32616, 32845, 32882] avg=32768.0000 max=32917 max_rank=1 min=32575 min_rank=0 imbalance_ratio=1.004547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=2 tokens_per_ep_rank=[32575, 32917, 32775, 32667, 32867, 32616, 32845, 32882] avg=32768.0000 max=32917 max_rank=1 min=32575 min_rank=0 imbalance_ratio=1.004547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=1 tokens_per_ep_rank=[32863, 32801, 32734, 32630, 32826, 32823, 32659, 32808] avg=32768.0000 max=32863 max_rank=0 min=32630 min_rank=3 imbalance_ratio=1.002899 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=2 tokens_per_ep_rank=[32575, 32917, 32775, 32667, 32867, 32616, 32845, 32882] avg=32768.0000 max=32917 max_rank=1 min=32575 min_rank=0 imbalance_ratio=1.004547 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=2 tokens_per_ep_rank=[32575, 32917, 32775, 32667, 32867, 32616, 32845, 32882] avg=32768.0000 max=32917 max_rank=1 min=32575 min_rank=0 imbalance_ratio=1.004547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=2 tokens_per_ep_rank=[32575, 32917, 32775, 32667, 32867, 32616, 32845, 32882] avg=32768.0000 max=32917 max_rank=1 min=32575 min_rank=0 imbalance_ratio=1.004547 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=2 tokens_per_ep_rank=[32575, 32917, 32775, 32667, 32867, 32616, 32845, 32882] avg=32768.0000 max=32917 max_rank=1 min=32575 min_rank=0 imbalance_ratio=1.004547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=2 tokens_per_ep_rank=[32575, 32917, 32775, 32667, 32867, 32616, 32845, 32882] avg=32768.0000 max=32917 max_rank=1 min=32575 min_rank=0 imbalance_ratio=1.004547 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=2 tokens_per_ep_rank=[32575, 32917, 32775, 32667, 32867, 32616, 32845, 32882] avg=32768.0000 max=32917 max_rank=1 min=32575 min_rank=0 imbalance_ratio=1.004547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=2 tokens_per_ep_rank=[32575, 32917, 32775, 32667, 32867, 32616, 32845, 32882] avg=32768.0000 max=32917 max_rank=1 min=32575 min_rank=0 imbalance_ratio=1.004547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=2 tokens_per_ep_rank=[32575, 32917, 32775, 32667, 32867, 32616, 32845, 32882] avg=32768.0000 max=32917 max_rank=1 min=32575 min_rank=0 imbalance_ratio=1.004547 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=2 tokens_per_ep_rank=[32575, 32917, 32775, 32667, 32867, 32616, 32845, 32882] avg=32768.0000 max=32917 max_rank=1 min=32575 min_rank=0 imbalance_ratio=1.004547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=2 tokens_per_ep_rank=[32575, 32917, 32775, 32667, 32867, 32616, 32845, 32882] avg=32768.0000 max=32917 max_rank=1 min=32575 min_rank=0 imbalance_ratio=1.004547 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=3 tokens_per_ep_rank=[32392, 32752, 32540, 32818, 32911, 32894, 32883, 32954] avg=32768.0000 max=32954 max_rank=7 min=32392 min_rank=0 imbalance_ratio=1.005676 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=2 tokens_per_ep_rank=[32575, 32917, 32775, 32667, 32867, 32616, 32845, 32882] avg=32768.0000 max=32917 max_rank=1 min=32575 min_rank=0 imbalance_ratio=1.004547 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=2 tokens_per_ep_rank=[32575, 32917, 32775, 32667, 32867, 32616, 32845, 32882] avg=32768.0000 max=32917 max_rank=1 min=32575 min_rank=0 imbalance_ratio=1.004547 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=3 tokens_per_ep_rank=[32392, 32752, 32540, 32818, 32911, 32894, 32883, 32954] avg=32768.0000 max=32954 max_rank=7 min=32392 min_rank=0 imbalance_ratio=1.005676 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=3 tokens_per_ep_rank=[32392, 32752, 32540, 32818, 32911, 32894, 32883, 32954] avg=32768.0000 max=32954 max_rank=7 min=32392 min_rank=0 imbalance_ratio=1.005676 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=2 tokens_per_ep_rank=[32575, 32917, 32775, 32667, 32867, 32616, 32845, 32882] avg=32768.0000 max=32917 max_rank=1 min=32575 min_rank=0 imbalance_ratio=1.004547 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=3 tokens_per_ep_rank=[32392, 32752, 32540, 32818, 32911, 32894, 32883, 32954] avg=32768.0000 max=32954 max_rank=7 min=32392 min_rank=0 imbalance_ratio=1.005676 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=3 tokens_per_ep_rank=[32392, 32752, 32540, 32818, 32911, 32894, 32883, 32954] avg=32768.0000 max=32954 max_rank=7 min=32392 min_rank=0 imbalance_ratio=1.005676 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=3 tokens_per_ep_rank=[32392, 32752, 32540, 32818, 32911, 32894, 32883, 32954] avg=32768.0000 max=32954 max_rank=7 min=32392 min_rank=0 imbalance_ratio=1.005676 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=3 tokens_per_ep_rank=[32392, 32752, 32540, 32818, 32911, 32894, 32883, 32954] avg=32768.0000 max=32954 max_rank=7 min=32392 min_rank=0 imbalance_ratio=1.005676 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=3 tokens_per_ep_rank=[32392, 32752, 32540, 32818, 32911, 32894, 32883, 32954] avg=32768.0000 max=32954 max_rank=7 min=32392 min_rank=0 imbalance_ratio=1.005676 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=3 tokens_per_ep_rank=[32392, 32752, 32540, 32818, 32911, 32894, 32883, 32954] avg=32768.0000 max=32954 max_rank=7 min=32392 min_rank=0 imbalance_ratio=1.005676 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=3 tokens_per_ep_rank=[32392, 32752, 32540, 32818, 32911, 32894, 32883, 32954] avg=32768.0000 max=32954 max_rank=7 min=32392 min_rank=0 imbalance_ratio=1.005676 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=3 tokens_per_ep_rank=[32392, 32752, 32540, 32818, 32911, 32894, 32883, 32954] avg=32768.0000 max=32954 max_rank=7 min=32392 min_rank=0 imbalance_ratio=1.005676 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=3 tokens_per_ep_rank=[32392, 32752, 32540, 32818, 32911, 32894, 32883, 32954] avg=32768.0000 max=32954 max_rank=7 min=32392 min_rank=0 imbalance_ratio=1.005676 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=3 tokens_per_ep_rank=[32392, 32752, 32540, 32818, 32911, 32894, 32883, 32954] avg=32768.0000 max=32954 max_rank=7 min=32392 min_rank=0 imbalance_ratio=1.005676 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=4 tokens_per_ep_rank=[32847, 32776, 33012, 32709, 32652, 32735, 32816, 32597] avg=32768.0000 max=33012 max_rank=2 min=32597 min_rank=7 imbalance_ratio=1.007446 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=3 tokens_per_ep_rank=[32392, 32752, 32540, 32818, 32911, 32894, 32883, 32954] avg=32768.0000 max=32954 max_rank=7 min=32392 min_rank=0 imbalance_ratio=1.005676 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=3 tokens_per_ep_rank=[32392, 32752, 32540, 32818, 32911, 32894, 32883, 32954] avg=32768.0000 max=32954 max_rank=7 min=32392 min_rank=0 imbalance_ratio=1.005676 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=4 tokens_per_ep_rank=[32847, 32776, 33012, 32709, 32652, 32735, 32816, 32597] avg=32768.0000 max=33012 max_rank=2 min=32597 min_rank=7 imbalance_ratio=1.007446 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=4 tokens_per_ep_rank=[32847, 32776, 33012, 32709, 32652, 32735, 32816, 32597] avg=32768.0000 max=33012 max_rank=2 min=32597 min_rank=7 imbalance_ratio=1.007446 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=3 tokens_per_ep_rank=[32392, 32752, 32540, 32818, 32911, 32894, 32883, 32954] avg=32768.0000 max=32954 max_rank=7 min=32392 min_rank=0 imbalance_ratio=1.005676 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=4 tokens_per_ep_rank=[32847, 32776, 33012, 32709, 32652, 32735, 32816, 32597] avg=32768.0000 max=33012 max_rank=2 min=32597 min_rank=7 imbalance_ratio=1.007446 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=4 tokens_per_ep_rank=[32847, 32776, 33012, 32709, 32652, 32735, 32816, 32597] avg=32768.0000 max=33012 max_rank=2 min=32597 min_rank=7 imbalance_ratio=1.007446 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=4 tokens_per_ep_rank=[32847, 32776, 33012, 32709, 32652, 32735, 32816, 32597] avg=32768.0000 max=33012 max_rank=2 min=32597 min_rank=7 imbalance_ratio=1.007446 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=4 tokens_per_ep_rank=[32847, 32776, 33012, 32709, 32652, 32735, 32816, 32597] avg=32768.0000 max=33012 max_rank=2 min=32597 min_rank=7 imbalance_ratio=1.007446 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=4 tokens_per_ep_rank=[32847, 32776, 33012, 32709, 32652, 32735, 32816, 32597] avg=32768.0000 max=33012 max_rank=2 min=32597 min_rank=7 imbalance_ratio=1.007446 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=4 tokens_per_ep_rank=[32847, 32776, 33012, 32709, 32652, 32735, 32816, 32597] avg=32768.0000 max=33012 max_rank=2 min=32597 min_rank=7 imbalance_ratio=1.007446 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=4 tokens_per_ep_rank=[32847, 32776, 33012, 32709, 32652, 32735, 32816, 32597] avg=32768.0000 max=33012 max_rank=2 min=32597 min_rank=7 imbalance_ratio=1.007446 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=4 tokens_per_ep_rank=[32847, 32776, 33012, 32709, 32652, 32735, 32816, 32597] avg=32768.0000 max=33012 max_rank=2 min=32597 min_rank=7 imbalance_ratio=1.007446 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=4 tokens_per_ep_rank=[32847, 32776, 33012, 32709, 32652, 32735, 32816, 32597] avg=32768.0000 max=33012 max_rank=2 min=32597 min_rank=7 imbalance_ratio=1.007446 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=4 tokens_per_ep_rank=[32847, 32776, 33012, 32709, 32652, 32735, 32816, 32597] avg=32768.0000 max=33012 max_rank=2 min=32597 min_rank=7 imbalance_ratio=1.007446 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=5 tokens_per_ep_rank=[32609, 32766, 32558, 32924, 32757, 32698, 32929, 32903] avg=32768.0000 max=32929 max_rank=6 min=32558 min_rank=2 imbalance_ratio=1.004913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=4 tokens_per_ep_rank=[32847, 32776, 33012, 32709, 32652, 32735, 32816, 32597] avg=32768.0000 max=33012 max_rank=2 min=32597 min_rank=7 imbalance_ratio=1.007446 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=5 tokens_per_ep_rank=[32609, 32766, 32558, 32924, 32757, 32698, 32929, 32903] avg=32768.0000 max=32929 max_rank=6 min=32558 min_rank=2 imbalance_ratio=1.004913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=4 tokens_per_ep_rank=[32847, 32776, 33012, 32709, 32652, 32735, 32816, 32597] avg=32768.0000 max=33012 max_rank=2 min=32597 min_rank=7 imbalance_ratio=1.007446 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=5 tokens_per_ep_rank=[32609, 32766, 32558, 32924, 32757, 32698, 32929, 32903] avg=32768.0000 max=32929 max_rank=6 min=32558 min_rank=2 imbalance_ratio=1.004913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=5 tokens_per_ep_rank=[32609, 32766, 32558, 32924, 32757, 32698, 32929, 32903] avg=32768.0000 max=32929 max_rank=6 min=32558 min_rank=2 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=5 tokens_per_ep_rank=[32609, 32766, 32558, 32924, 32757, 32698, 32929, 32903] avg=32768.0000 max=32929 max_rank=6 min=32558 min_rank=2 imbalance_ratio=1.004913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=4 tokens_per_ep_rank=[32847, 32776, 33012, 32709, 32652, 32735, 32816, 32597] avg=32768.0000 max=33012 max_rank=2 min=32597 min_rank=7 imbalance_ratio=1.007446 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=5 tokens_per_ep_rank=[32609, 32766, 32558, 32924, 32757, 32698, 32929, 32903] avg=32768.0000 max=32929 max_rank=6 min=32558 min_rank=2 imbalance_ratio=1.004913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=5 tokens_per_ep_rank=[32609, 32766, 32558, 32924, 32757, 32698, 32929, 32903] avg=32768.0000 max=32929 max_rank=6 min=32558 min_rank=2 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=5 tokens_per_ep_rank=[32609, 32766, 32558, 32924, 32757, 32698, 32929, 32903] avg=32768.0000 max=32929 max_rank=6 min=32558 min_rank=2 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=5 tokens_per_ep_rank=[32609, 32766, 32558, 32924, 32757, 32698, 32929, 32903] avg=32768.0000 max=32929 max_rank=6 min=32558 min_rank=2 imbalance_ratio=1.004913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=5 tokens_per_ep_rank=[32609, 32766, 32558, 32924, 32757, 32698, 32929, 32903] avg=32768.0000 max=32929 max_rank=6 min=32558 min_rank=2 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=5 tokens_per_ep_rank=[32609, 32766, 32558, 32924, 32757, 32698, 32929, 32903] avg=32768.0000 max=32929 max_rank=6 min=32558 min_rank=2 imbalance_ratio=1.004913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=6 tokens_per_ep_rank=[32711, 32763, 32591, 32901, 32862, 32796, 32654, 32866] avg=32768.0000 max=32901 max_rank=3 min=32591 min_rank=2 imbalance_ratio=1.004059 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=5 tokens_per_ep_rank=[32609, 32766, 32558, 32924, 32757, 32698, 32929, 32903] avg=32768.0000 max=32929 max_rank=6 min=32558 min_rank=2 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=5 tokens_per_ep_rank=[32609, 32766, 32558, 32924, 32757, 32698, 32929, 32903] avg=32768.0000 max=32929 max_rank=6 min=32558 min_rank=2 imbalance_ratio=1.004913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=5 tokens_per_ep_rank=[32609, 32766, 32558, 32924, 32757, 32698, 32929, 32903] avg=32768.0000 max=32929 max_rank=6 min=32558 min_rank=2 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=6 tokens_per_ep_rank=[32711, 32763, 32591, 32901, 32862, 32796, 32654, 32866] avg=32768.0000 max=32901 max_rank=3 min=32591 min_rank=2 imbalance_ratio=1.004059 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=5 tokens_per_ep_rank=[32609, 32766, 32558, 32924, 32757, 32698, 32929, 32903] avg=32768.0000 max=32929 max_rank=6 min=32558 min_rank=2 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=6 tokens_per_ep_rank=[32711, 32763, 32591, 32901, 32862, 32796, 32654, 32866] avg=32768.0000 max=32901 max_rank=3 min=32591 min_rank=2 imbalance_ratio=1.004059 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=6 tokens_per_ep_rank=[32711, 32763, 32591, 32901, 32862, 32796, 32654, 32866] avg=32768.0000 max=32901 max_rank=3 min=32591 min_rank=2 imbalance_ratio=1.004059 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=6 tokens_per_ep_rank=[32711, 32763, 32591, 32901, 32862, 32796, 32654, 32866] avg=32768.0000 max=32901 max_rank=3 min=32591 min_rank=2 imbalance_ratio=1.004059 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=5 tokens_per_ep_rank=[32609, 32766, 32558, 32924, 32757, 32698, 32929, 32903] avg=32768.0000 max=32929 max_rank=6 min=32558 min_rank=2 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=6 tokens_per_ep_rank=[32711, 32763, 32591, 32901, 32862, 32796, 32654, 32866] avg=32768.0000 max=32901 max_rank=3 min=32591 min_rank=2 imbalance_ratio=1.004059 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=6 tokens_per_ep_rank=[32711, 32763, 32591, 32901, 32862, 32796, 32654, 32866] avg=32768.0000 max=32901 max_rank=3 min=32591 min_rank=2 imbalance_ratio=1.004059 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=6 tokens_per_ep_rank=[32711, 32763, 32591, 32901, 32862, 32796, 32654, 32866] avg=32768.0000 max=32901 max_rank=3 min=32591 min_rank=2 imbalance_ratio=1.004059 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=6 tokens_per_ep_rank=[32711, 32763, 32591, 32901, 32862, 32796, 32654, 32866] avg=32768.0000 max=32901 max_rank=3 min=32591 min_rank=2 imbalance_ratio=1.004059 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=6 tokens_per_ep_rank=[32711, 32763, 32591, 32901, 32862, 32796, 32654, 32866] avg=32768.0000 max=32901 max_rank=3 min=32591 min_rank=2 imbalance_ratio=1.004059 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=6 tokens_per_ep_rank=[32711, 32763, 32591, 32901, 32862, 32796, 32654, 32866] avg=32768.0000 max=32901 max_rank=3 min=32591 min_rank=2 imbalance_ratio=1.004059 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=7 tokens_per_ep_rank=[32813, 32699, 32594, 32780, 32762, 32607, 32967, 32922] avg=32768.0000 max=32967 max_rank=6 min=32594 min_rank=2 imbalance_ratio=1.006073 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=6 tokens_per_ep_rank=[32711, 32763, 32591, 32901, 32862, 32796, 32654, 32866] avg=32768.0000 max=32901 max_rank=3 min=32591 min_rank=2 imbalance_ratio=1.004059 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=6 tokens_per_ep_rank=[32711, 32763, 32591, 32901, 32862, 32796, 32654, 32866] avg=32768.0000 max=32901 max_rank=3 min=32591 min_rank=2 imbalance_ratio=1.004059 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=6 tokens_per_ep_rank=[32711, 32763, 32591, 32901, 32862, 32796, 32654, 32866] avg=32768.0000 max=32901 max_rank=3 min=32591 min_rank=2 imbalance_ratio=1.004059 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=7 tokens_per_ep_rank=[32813, 32699, 32594, 32780, 32762, 32607, 32967, 32922] avg=32768.0000 max=32967 max_rank=6 min=32594 min_rank=2 imbalance_ratio=1.006073 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=6 tokens_per_ep_rank=[32711, 32763, 32591, 32901, 32862, 32796, 32654, 32866] avg=32768.0000 max=32901 max_rank=3 min=32591 min_rank=2 imbalance_ratio=1.004059 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=7 tokens_per_ep_rank=[32813, 32699, 32594, 32780, 32762, 32607, 32967, 32922] avg=32768.0000 max=32967 max_rank=6 min=32594 min_rank=2 imbalance_ratio=1.006073 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=7 tokens_per_ep_rank=[32813, 32699, 32594, 32780, 32762, 32607, 32967, 32922] avg=32768.0000 max=32967 max_rank=6 min=32594 min_rank=2 imbalance_ratio=1.006073 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=7 tokens_per_ep_rank=[32813, 32699, 32594, 32780, 32762, 32607, 32967, 32922] avg=32768.0000 max=32967 max_rank=6 min=32594 min_rank=2 imbalance_ratio=1.006073 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=6 tokens_per_ep_rank=[32711, 32763, 32591, 32901, 32862, 32796, 32654, 32866] avg=32768.0000 max=32901 max_rank=3 min=32591 min_rank=2 imbalance_ratio=1.004059 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=7 tokens_per_ep_rank=[32813, 32699, 32594, 32780, 32762, 32607, 32967, 32922] avg=32768.0000 max=32967 max_rank=6 min=32594 min_rank=2 imbalance_ratio=1.006073 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=7 tokens_per_ep_rank=[32813, 32699, 32594, 32780, 32762, 32607, 32967, 32922] avg=32768.0000 max=32967 max_rank=6 min=32594 min_rank=2 imbalance_ratio=1.006073 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=7 tokens_per_ep_rank=[32813, 32699, 32594, 32780, 32762, 32607, 32967, 32922] avg=32768.0000 max=32967 max_rank=6 min=32594 min_rank=2 imbalance_ratio=1.006073 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=7 tokens_per_ep_rank=[32813, 32699, 32594, 32780, 32762, 32607, 32967, 32922] avg=32768.0000 max=32967 max_rank=6 min=32594 min_rank=2 imbalance_ratio=1.006073 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=7 tokens_per_ep_rank=[32813, 32699, 32594, 32780, 32762, 32607, 32967, 32922] avg=32768.0000 max=32967 max_rank=6 min=32594 min_rank=2 imbalance_ratio=1.006073 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=7 tokens_per_ep_rank=[32813, 32699, 32594, 32780, 32762, 32607, 32967, 32922] avg=32768.0000 max=32967 max_rank=6 min=32594 min_rank=2 imbalance_ratio=1.006073 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=8 tokens_per_ep_rank=[33053, 32578, 32736, 32828, 32899, 32638, 32509, 32903] avg=32768.0000 max=33053 max_rank=0 min=32509 min_rank=6 imbalance_ratio=1.008698 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=7 tokens_per_ep_rank=[32813, 32699, 32594, 32780, 32762, 32607, 32967, 32922] avg=32768.0000 max=32967 max_rank=6 min=32594 min_rank=2 imbalance_ratio=1.006073 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=7 tokens_per_ep_rank=[32813, 32699, 32594, 32780, 32762, 32607, 32967, 32922] avg=32768.0000 max=32967 max_rank=6 min=32594 min_rank=2 imbalance_ratio=1.006073 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=7 tokens_per_ep_rank=[32813, 32699, 32594, 32780, 32762, 32607, 32967, 32922] avg=32768.0000 max=32967 max_rank=6 min=32594 min_rank=2 imbalance_ratio=1.006073 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=8 tokens_per_ep_rank=[33053, 32578, 32736, 32828, 32899, 32638, 32509, 32903] avg=32768.0000 max=33053 max_rank=0 min=32509 min_rank=6 imbalance_ratio=1.008698 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=7 tokens_per_ep_rank=[32813, 32699, 32594, 32780, 32762, 32607, 32967, 32922] avg=32768.0000 max=32967 max_rank=6 min=32594 min_rank=2 imbalance_ratio=1.006073 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=8 tokens_per_ep_rank=[33053, 32578, 32736, 32828, 32899, 32638, 32509, 32903] avg=32768.0000 max=33053 max_rank=0 min=32509 min_rank=6 imbalance_ratio=1.008698 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=8 tokens_per_ep_rank=[33053, 32578, 32736, 32828, 32899, 32638, 32509, 32903] avg=32768.0000 max=33053 max_rank=0 min=32509 min_rank=6 imbalance_ratio=1.008698 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=8 tokens_per_ep_rank=[33053, 32578, 32736, 32828, 32899, 32638, 32509, 32903] avg=32768.0000 max=33053 max_rank=0 min=32509 min_rank=6 imbalance_ratio=1.008698 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=7 tokens_per_ep_rank=[32813, 32699, 32594, 32780, 32762, 32607, 32967, 32922] avg=32768.0000 max=32967 max_rank=6 min=32594 min_rank=2 imbalance_ratio=1.006073 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=8 tokens_per_ep_rank=[33053, 32578, 32736, 32828, 32899, 32638, 32509, 32903] avg=32768.0000 max=33053 max_rank=0 min=32509 min_rank=6 imbalance_ratio=1.008698 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=8 tokens_per_ep_rank=[33053, 32578, 32736, 32828, 32899, 32638, 32509, 32903] avg=32768.0000 max=33053 max_rank=0 min=32509 min_rank=6 imbalance_ratio=1.008698 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=8 tokens_per_ep_rank=[33053, 32578, 32736, 32828, 32899, 32638, 32509, 32903] avg=32768.0000 max=33053 max_rank=0 min=32509 min_rank=6 imbalance_ratio=1.008698 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=8 tokens_per_ep_rank=[33053, 32578, 32736, 32828, 32899, 32638, 32509, 32903] avg=32768.0000 max=33053 max_rank=0 min=32509 min_rank=6 imbalance_ratio=1.008698 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=8 tokens_per_ep_rank=[33053, 32578, 32736, 32828, 32899, 32638, 32509, 32903] avg=32768.0000 max=33053 max_rank=0 min=32509 min_rank=6 imbalance_ratio=1.008698 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=9 tokens_per_ep_rank=[32678, 32880, 32830, 32764, 32963, 32870, 32425, 32734] avg=32768.0000 max=32963 max_rank=4 min=32425 min_rank=6 imbalance_ratio=1.005951 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=8 tokens_per_ep_rank=[33053, 32578, 32736, 32828, 32899, 32638, 32509, 32903] avg=32768.0000 max=33053 max_rank=0 min=32509 min_rank=6 imbalance_ratio=1.008698 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=8 tokens_per_ep_rank=[33053, 32578, 32736, 32828, 32899, 32638, 32509, 32903] avg=32768.0000 max=33053 max_rank=0 min=32509 min_rank=6 imbalance_ratio=1.008698 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=8 tokens_per_ep_rank=[33053, 32578, 32736, 32828, 32899, 32638, 32509, 32903] avg=32768.0000 max=33053 max_rank=0 min=32509 min_rank=6 imbalance_ratio=1.008698 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=9 tokens_per_ep_rank=[32678, 32880, 32830, 32764, 32963, 32870, 32425, 32734] avg=32768.0000 max=32963 max_rank=4 min=32425 min_rank=6 imbalance_ratio=1.005951 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=8 tokens_per_ep_rank=[33053, 32578, 32736, 32828, 32899, 32638, 32509, 32903] avg=32768.0000 max=33053 max_rank=0 min=32509 min_rank=6 imbalance_ratio=1.008698 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=9 tokens_per_ep_rank=[32678, 32880, 32830, 32764, 32963, 32870, 32425, 32734] avg=32768.0000 max=32963 max_rank=4 min=32425 min_rank=6 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=8 tokens_per_ep_rank=[33053, 32578, 32736, 32828, 32899, 32638, 32509, 32903] avg=32768.0000 max=33053 max_rank=0 min=32509 min_rank=6 imbalance_ratio=1.008698 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=9 tokens_per_ep_rank=[32678, 32880, 32830, 32764, 32963, 32870, 32425, 32734] avg=32768.0000 max=32963 max_rank=4 min=32425 min_rank=6 imbalance_ratio=1.005951 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=9 tokens_per_ep_rank=[32678, 32880, 32830, 32764, 32963, 32870, 32425, 32734] avg=32768.0000 max=32963 max_rank=4 min=32425 min_rank=6 imbalance_ratio=1.005951 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=8 tokens_per_ep_rank=[33053, 32578, 32736, 32828, 32899, 32638, 32509, 32903] avg=32768.0000 max=33053 max_rank=0 min=32509 min_rank=6 imbalance_ratio=1.008698 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=9 tokens_per_ep_rank=[32678, 32880, 32830, 32764, 32963, 32870, 32425, 32734] avg=32768.0000 max=32963 max_rank=4 min=32425 min_rank=6 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=9 tokens_per_ep_rank=[32678, 32880, 32830, 32764, 32963, 32870, 32425, 32734] avg=32768.0000 max=32963 max_rank=4 min=32425 min_rank=6 imbalance_ratio=1.005951 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=9 tokens_per_ep_rank=[32678, 32880, 32830, 32764, 32963, 32870, 32425, 32734] avg=32768.0000 max=32963 max_rank=4 min=32425 min_rank=6 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=9 tokens_per_ep_rank=[32678, 32880, 32830, 32764, 32963, 32870, 32425, 32734] avg=32768.0000 max=32963 max_rank=4 min=32425 min_rank=6 imbalance_ratio=1.005951 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=9 tokens_per_ep_rank=[32678, 32880, 32830, 32764, 32963, 32870, 32425, 32734] avg=32768.0000 max=32963 max_rank=4 min=32425 min_rank=6 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=10 tokens_per_ep_rank=[32562, 33011, 32494, 33010, 32749, 32841, 32925, 32552] avg=32768.0000 max=33011 max_rank=1 min=32494 min_rank=2 imbalance_ratio=1.007416 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=9 tokens_per_ep_rank=[32678, 32880, 32830, 32764, 32963, 32870, 32425, 32734] avg=32768.0000 max=32963 max_rank=4 min=32425 min_rank=6 imbalance_ratio=1.005951 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=9 tokens_per_ep_rank=[32678, 32880, 32830, 32764, 32963, 32870, 32425, 32734] avg=32768.0000 max=32963 max_rank=4 min=32425 min_rank=6 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=10 tokens_per_ep_rank=[32562, 33011, 32494, 33010, 32749, 32841, 32925, 32552] avg=32768.0000 max=33011 max_rank=1 min=32494 min_rank=2 imbalance_ratio=1.007416 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=9 tokens_per_ep_rank=[32678, 32880, 32830, 32764, 32963, 32870, 32425, 32734] avg=32768.0000 max=32963 max_rank=4 min=32425 min_rank=6 imbalance_ratio=1.005951 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=9 tokens_per_ep_rank=[32678, 32880, 32830, 32764, 32963, 32870, 32425, 32734] avg=32768.0000 max=32963 max_rank=4 min=32425 min_rank=6 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=10 tokens_per_ep_rank=[32562, 33011, 32494, 33010, 32749, 32841, 32925, 32552] avg=32768.0000 max=33011 max_rank=1 min=32494 min_rank=2 imbalance_ratio=1.007416 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=10 tokens_per_ep_rank=[32562, 33011, 32494, 33010, 32749, 32841, 32925, 32552] avg=32768.0000 max=33011 max_rank=1 min=32494 min_rank=2 imbalance_ratio=1.007416 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=9 tokens_per_ep_rank=[32678, 32880, 32830, 32764, 32963, 32870, 32425, 32734] avg=32768.0000 max=32963 max_rank=4 min=32425 min_rank=6 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=10 tokens_per_ep_rank=[32562, 33011, 32494, 33010, 32749, 32841, 32925, 32552] avg=32768.0000 max=33011 max_rank=1 min=32494 min_rank=2 imbalance_ratio=1.007416 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=10 tokens_per_ep_rank=[32562, 33011, 32494, 33010, 32749, 32841, 32925, 32552] avg=32768.0000 max=33011 max_rank=1 min=32494 min_rank=2 imbalance_ratio=1.007416 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=9 tokens_per_ep_rank=[32678, 32880, 32830, 32764, 32963, 32870, 32425, 32734] avg=32768.0000 max=32963 max_rank=4 min=32425 min_rank=6 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=10 tokens_per_ep_rank=[32562, 33011, 32494, 33010, 32749, 32841, 32925, 32552] avg=32768.0000 max=33011 max_rank=1 min=32494 min_rank=2 imbalance_ratio=1.007416 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=10 tokens_per_ep_rank=[32562, 33011, 32494, 33010, 32749, 32841, 32925, 32552] avg=32768.0000 max=33011 max_rank=1 min=32494 min_rank=2 imbalance_ratio=1.007416 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=10 tokens_per_ep_rank=[32562, 33011, 32494, 33010, 32749, 32841, 32925, 32552] avg=32768.0000 max=33011 max_rank=1 min=32494 min_rank=2 imbalance_ratio=1.007416 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=10 tokens_per_ep_rank=[32562, 33011, 32494, 33010, 32749, 32841, 32925, 32552] avg=32768.0000 max=33011 max_rank=1 min=32494 min_rank=2 imbalance_ratio=1.007416 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=10 tokens_per_ep_rank=[32562, 33011, 32494, 33010, 32749, 32841, 32925, 32552] avg=32768.0000 max=33011 max_rank=1 min=32494 min_rank=2 imbalance_ratio=1.007416 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=10 tokens_per_ep_rank=[32562, 33011, 32494, 33010, 32749, 32841, 32925, 32552] avg=32768.0000 max=33011 max_rank=1 min=32494 min_rank=2 imbalance_ratio=1.007416 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=10 tokens_per_ep_rank=[32562, 33011, 32494, 33010, 32749, 32841, 32925, 32552] avg=32768.0000 max=33011 max_rank=1 min=32494 min_rank=2 imbalance_ratio=1.007416 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=10 tokens_per_ep_rank=[32562, 33011, 32494, 33010, 32749, 32841, 32925, 32552] avg=32768.0000 max=33011 max_rank=1 min=32494 min_rank=2 imbalance_ratio=1.007416 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=10 tokens_per_ep_rank=[32562, 33011, 32494, 33010, 32749, 32841, 32925, 32552] avg=32768.0000 max=33011 max_rank=1 min=32494 min_rank=2 imbalance_ratio=1.007416 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=10 tokens_per_ep_rank=[32562, 33011, 32494, 33010, 32749, 32841, 32925, 32552] avg=32768.0000 max=33011 max_rank=1 min=32494 min_rank=2 imbalance_ratio=1.007416 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=10 records=10 total_expected_recv=327518 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=77056 peak_demand=32894 required_capacity=1.003845 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=10 records=10 total_expected_recv=328121 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76544 peak_demand=32954 required_capacity=1.005676 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=10 records=10 total_expected_recv=328248 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76544 peak_demand=32963 required_capacity=1.005951 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=10 records=10 total_expected_recv=326864 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=78208 peak_demand=33012 required_capacity=1.007446 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=10 records=10 total_expected_recv=327612 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=77056 peak_demand=32967 required_capacity=1.006073 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=10 records=10 total_expected_recv=327943 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76800 peak_demand=33011 required_capacity=1.007416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=10 records=10 total_expected_recv=328031 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76032 peak_demand=33010 required_capacity=1.007385 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=10 records=10 total_expected_recv=327103 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=77056 peak_demand=33053 required_capacity=1.008698 + [2026-08-26 15:53:14] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 607.9 | throughput per GPU (TFLOP/s/GPU): 203.1 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.509129E+00 | loss scale: 1.0 | grad norm: 3.494 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 33664 | min safe capacity: 1.0273 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=0 calls=1 budget=40960 expected_recv=32621 expected_overflow=0 exact_recv_per_rank=[32880, 32719, 32613, 32940, 32759, 32972, 32640, 32621] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=0 calls=1 budget=40960 expected_recv=32759 expected_overflow=0 exact_recv_per_rank=[32880, 32719, 32613, 32940, 32759, 32972, 32640, 32621] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=0 calls=1 budget=40960 expected_recv=32719 expected_overflow=0 exact_recv_per_rank=[32880, 32719, 32613, 32940, 32759, 32972, 32640, 32621] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=0 calls=1 budget=40960 expected_recv=32613 expected_overflow=0 exact_recv_per_rank=[32880, 32719, 32613, 32940, 32759, 32972, 32640, 32621] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=0 calls=1 budget=40960 expected_recv=32640 expected_overflow=0 exact_recv_per_rank=[32880, 32719, 32613, 32940, 32759, 32972, 32640, 32621] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=0 calls=1 budget=40960 expected_recv=32972 expected_overflow=0 exact_recv_per_rank=[32880, 32719, 32613, 32940, 32759, 32972, 32640, 32621] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=0 calls=1 budget=40960 expected_recv=32940 expected_overflow=0 exact_recv_per_rank=[32880, 32719, 32613, 32940, 32759, 32972, 32640, 32621] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=0 calls=1 budget=40960 expected_recv=32621 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=0 calls=1 budget=40960 expected_recv=32759 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=0 calls=1 budget=40960 expected_recv=32719 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=0 calls=1 budget=40960 expected_recv=32880 expected_overflow=0 exact_recv_per_rank=[32880, 32719, 32613, 32940, 32759, 32972, 32640, 32621] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=0 calls=1 budget=40960 expected_recv=32613 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=0 calls=1 budget=40960 expected_recv=32640 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=0 calls=1 budget=40960 expected_recv=32972 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=0 calls=1 budget=40960 expected_recv=32940 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=0 calls=1 budget=40960 expected_recv=32880 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=1 calls=1 budget=40960 expected_recv=32779 expected_overflow=0 exact_recv_per_rank=[32953, 32778, 32810, 32891, 32779, 32776, 32435, 32722] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=1 calls=1 budget=40960 expected_recv=32722 expected_overflow=0 exact_recv_per_rank=[32953, 32778, 32810, 32891, 32779, 32776, 32435, 32722] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=1 calls=1 budget=40960 expected_recv=32778 expected_overflow=0 exact_recv_per_rank=[32953, 32778, 32810, 32891, 32779, 32776, 32435, 32722] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=1 calls=1 budget=40960 expected_recv=32779 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=1 calls=1 budget=40960 expected_recv=32722 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=1 calls=1 budget=40960 expected_recv=32778 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=1 calls=1 budget=40960 expected_recv=32810 expected_overflow=0 exact_recv_per_rank=[32953, 32778, 32810, 32891, 32779, 32776, 32435, 32722] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=1 calls=1 budget=40960 expected_recv=32435 expected_overflow=0 exact_recv_per_rank=[32953, 32778, 32810, 32891, 32779, 32776, 32435, 32722] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=1 calls=1 budget=40960 expected_recv=32776 expected_overflow=0 exact_recv_per_rank=[32953, 32778, 32810, 32891, 32779, 32776, 32435, 32722] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=1 calls=1 budget=40960 expected_recv=32810 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=1 calls=1 budget=40960 expected_recv=32435 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=1 calls=1 budget=40960 expected_recv=32891 expected_overflow=0 exact_recv_per_rank=[32953, 32778, 32810, 32891, 32779, 32776, 32435, 32722] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=1 calls=1 budget=40960 expected_recv=32953 expected_overflow=0 exact_recv_per_rank=[32953, 32778, 32810, 32891, 32779, 32776, 32435, 32722] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=1 calls=1 budget=40960 expected_recv=32776 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=1 calls=1 budget=40960 expected_recv=32891 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=1 calls=1 budget=40960 expected_recv=32953 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=2 calls=1 budget=40960 expected_recv=32694 expected_overflow=0 exact_recv_per_rank=[32795, 32739, 32741, 33056, 32694, 32642, 32686, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=2 calls=1 budget=40960 expected_recv=32791 expected_overflow=0 exact_recv_per_rank=[32795, 32739, 32741, 33056, 32694, 32642, 32686, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=2 calls=1 budget=40960 expected_recv=32739 expected_overflow=0 exact_recv_per_rank=[32795, 32739, 32741, 33056, 32694, 32642, 32686, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=2 calls=1 budget=40960 expected_recv=32694 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=2 calls=1 budget=40960 expected_recv=32791 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=2 calls=1 budget=40960 expected_recv=32739 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=2 calls=1 budget=40960 expected_recv=32741 expected_overflow=0 exact_recv_per_rank=[32795, 32739, 32741, 33056, 32694, 32642, 32686, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=2 calls=1 budget=40960 expected_recv=32686 expected_overflow=0 exact_recv_per_rank=[32795, 32739, 32741, 33056, 32694, 32642, 32686, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=2 calls=1 budget=40960 expected_recv=32642 expected_overflow=0 exact_recv_per_rank=[32795, 32739, 32741, 33056, 32694, 32642, 32686, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=2 calls=1 budget=40960 expected_recv=32795 expected_overflow=0 exact_recv_per_rank=[32795, 32739, 32741, 33056, 32694, 32642, 32686, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=2 calls=1 budget=40960 expected_recv=33056 expected_overflow=0 exact_recv_per_rank=[32795, 32739, 32741, 33056, 32694, 32642, 32686, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=2 calls=1 budget=40960 expected_recv=32741 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=2 calls=1 budget=40960 expected_recv=32686 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=2 calls=1 budget=40960 expected_recv=32642 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=2 calls=1 budget=40960 expected_recv=32795 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=2 calls=1 budget=40960 expected_recv=33056 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=3 calls=1 budget=40960 expected_recv=33015 expected_overflow=0 exact_recv_per_rank=[32934, 32560, 32645, 32565, 32904, 32463, 33058, 33015] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=3 calls=1 budget=40960 expected_recv=32904 expected_overflow=0 exact_recv_per_rank=[32934, 32560, 32645, 32565, 32904, 32463, 33058, 33015] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=3 calls=1 budget=40960 expected_recv=33015 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=3 calls=1 budget=40960 expected_recv=32560 expected_overflow=0 exact_recv_per_rank=[32934, 32560, 32645, 32565, 32904, 32463, 33058, 33015] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=3 calls=1 budget=40960 expected_recv=32904 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=3 calls=1 budget=40960 expected_recv=32560 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=3 calls=1 budget=40960 expected_recv=32645 expected_overflow=0 exact_recv_per_rank=[32934, 32560, 32645, 32565, 32904, 32463, 33058, 33015] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=3 calls=1 budget=40960 expected_recv=33058 expected_overflow=0 exact_recv_per_rank=[32934, 32560, 32645, 32565, 32904, 32463, 33058, 33015] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=3 calls=1 budget=40960 expected_recv=32463 expected_overflow=0 exact_recv_per_rank=[32934, 32560, 32645, 32565, 32904, 32463, 33058, 33015] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=3 calls=1 budget=40960 expected_recv=32934 expected_overflow=0 exact_recv_per_rank=[32934, 32560, 32645, 32565, 32904, 32463, 33058, 33015] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=3 calls=1 budget=40960 expected_recv=32565 expected_overflow=0 exact_recv_per_rank=[32934, 32560, 32645, 32565, 32904, 32463, 33058, 33015] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=3 calls=1 budget=40960 expected_recv=32645 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=3 calls=1 budget=40960 expected_recv=33058 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=3 calls=1 budget=40960 expected_recv=32463 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=3 calls=1 budget=40960 expected_recv=32934 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=3 calls=1 budget=40960 expected_recv=32565 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=4 calls=1 budget=40960 expected_recv=32857 expected_overflow=0 exact_recv_per_rank=[32844, 32780, 32709, 32780, 32865, 32737, 32572, 32857] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=4 calls=1 budget=40960 expected_recv=32865 expected_overflow=0 exact_recv_per_rank=[32844, 32780, 32709, 32780, 32865, 32737, 32572, 32857] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=4 calls=1 budget=40960 expected_recv=32857 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=4 calls=1 budget=40960 expected_recv=32865 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=4 calls=1 budget=40960 expected_recv=32780 expected_overflow=0 exact_recv_per_rank=[32844, 32780, 32709, 32780, 32865, 32737, 32572, 32857] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=4 calls=1 budget=40960 expected_recv=32780 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=4 calls=1 budget=40960 expected_recv=32709 expected_overflow=0 exact_recv_per_rank=[32844, 32780, 32709, 32780, 32865, 32737, 32572, 32857] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=4 calls=1 budget=40960 expected_recv=32737 expected_overflow=0 exact_recv_per_rank=[32844, 32780, 32709, 32780, 32865, 32737, 32572, 32857] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=4 calls=1 budget=40960 expected_recv=32572 expected_overflow=0 exact_recv_per_rank=[32844, 32780, 32709, 32780, 32865, 32737, 32572, 32857] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=4 calls=1 budget=40960 expected_recv=32709 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=4 calls=1 budget=40960 expected_recv=32844 expected_overflow=0 exact_recv_per_rank=[32844, 32780, 32709, 32780, 32865, 32737, 32572, 32857] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=4 calls=1 budget=40960 expected_recv=32780 expected_overflow=0 exact_recv_per_rank=[32844, 32780, 32709, 32780, 32865, 32737, 32572, 32857] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=4 calls=1 budget=40960 expected_recv=32737 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=4 calls=1 budget=40960 expected_recv=32572 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=4 calls=1 budget=40960 expected_recv=32844 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=4 calls=1 budget=40960 expected_recv=32780 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=5 calls=1 budget=40960 expected_recv=32775 expected_overflow=0 exact_recv_per_rank=[32623, 32968, 33096, 32472, 32775, 32550, 33096, 32564] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=5 calls=1 budget=40960 expected_recv=32564 expected_overflow=0 exact_recv_per_rank=[32623, 32968, 33096, 32472, 32775, 32550, 33096, 32564] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=5 calls=1 budget=40960 expected_recv=32775 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=5 calls=1 budget=40960 expected_recv=32564 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=5 calls=1 budget=40960 expected_recv=32968 expected_overflow=0 exact_recv_per_rank=[32623, 32968, 33096, 32472, 32775, 32550, 33096, 32564] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=5 calls=1 budget=40960 expected_recv=32968 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32623, 32968, 33096, 32472, 32775, 32550, 33096, 32564] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=5 calls=1 budget=40960 expected_recv=32550 expected_overflow=0 exact_recv_per_rank=[32623, 32968, 33096, 32472, 32775, 32550, 33096, 32564] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32623, 32968, 33096, 32472, 32775, 32550, 33096, 32564] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=5 calls=1 budget=40960 expected_recv=32623 expected_overflow=0 exact_recv_per_rank=[32623, 32968, 33096, 32472, 32775, 32550, 33096, 32564] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=5 calls=1 budget=40960 expected_recv=32472 expected_overflow=0 exact_recv_per_rank=[32623, 32968, 33096, 32472, 32775, 32550, 33096, 32564] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=5 calls=1 budget=40960 expected_recv=32550 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=5 calls=1 budget=40960 expected_recv=32623 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=5 calls=1 budget=40960 expected_recv=32472 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=6 calls=1 budget=40960 expected_recv=32731 expected_overflow=0 exact_recv_per_rank=[32618, 32655, 32850, 32829, 32730, 32635, 33096, 32731] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=6 calls=1 budget=40960 expected_recv=32730 expected_overflow=0 exact_recv_per_rank=[32618, 32655, 32850, 32829, 32730, 32635, 33096, 32731] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=6 calls=1 budget=40960 expected_recv=32731 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=6 calls=1 budget=40960 expected_recv=32655 expected_overflow=0 exact_recv_per_rank=[32618, 32655, 32850, 32829, 32730, 32635, 33096, 32731] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=6 calls=1 budget=40960 expected_recv=32730 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=6 calls=1 budget=40960 expected_recv=32655 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=6 calls=1 budget=40960 expected_recv=32850 expected_overflow=0 exact_recv_per_rank=[32618, 32655, 32850, 32829, 32730, 32635, 33096, 32731] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=6 calls=1 budget=40960 expected_recv=32635 expected_overflow=0 exact_recv_per_rank=[32618, 32655, 32850, 32829, 32730, 32635, 33096, 32731] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=6 calls=1 budget=40960 expected_recv=32618 expected_overflow=0 exact_recv_per_rank=[32618, 32655, 32850, 32829, 32730, 32635, 33096, 32731] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32618, 32655, 32850, 32829, 32730, 32635, 33096, 32731] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=6 calls=1 budget=40960 expected_recv=32850 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=6 calls=1 budget=40960 expected_recv=32829 expected_overflow=0 exact_recv_per_rank=[32618, 32655, 32850, 32829, 32730, 32635, 33096, 32731] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=6 calls=1 budget=40960 expected_recv=32635 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=6 calls=1 budget=40960 expected_recv=32618 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=7 calls=1 budget=40960 expected_recv=32712 expected_overflow=0 exact_recv_per_rank=[32890, 33065, 33068, 32516, 32441, 32708, 32744, 32712] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=6 calls=1 budget=40960 expected_recv=32829 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=7 calls=1 budget=40960 expected_recv=32441 expected_overflow=0 exact_recv_per_rank=[32890, 33065, 33068, 32516, 32441, 32708, 32744, 32712] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=7 calls=1 budget=40960 expected_recv=32712 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=7 calls=1 budget=40960 expected_recv=32441 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=7 calls=1 budget=40960 expected_recv=33065 expected_overflow=0 exact_recv_per_rank=[32890, 33065, 33068, 32516, 32441, 32708, 32744, 32712] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=7 calls=1 budget=40960 expected_recv=33068 expected_overflow=0 exact_recv_per_rank=[32890, 33065, 33068, 32516, 32441, 32708, 32744, 32712] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=7 calls=1 budget=40960 expected_recv=33065 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=7 calls=1 budget=40960 expected_recv=32708 expected_overflow=0 exact_recv_per_rank=[32890, 33065, 33068, 32516, 32441, 32708, 32744, 32712] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=7 calls=1 budget=40960 expected_recv=32890 expected_overflow=0 exact_recv_per_rank=[32890, 33065, 33068, 32516, 32441, 32708, 32744, 32712] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=7 calls=1 budget=40960 expected_recv=32744 expected_overflow=0 exact_recv_per_rank=[32890, 33065, 33068, 32516, 32441, 32708, 32744, 32712] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=7 calls=1 budget=40960 expected_recv=33068 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=7 calls=1 budget=40960 expected_recv=32708 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=7 calls=1 budget=40960 expected_recv=32890 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=7 calls=1 budget=40960 expected_recv=32744 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=7 calls=1 budget=40960 expected_recv=32516 expected_overflow=0 exact_recv_per_rank=[32890, 33065, 33068, 32516, 32441, 32708, 32744, 32712] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=8 calls=1 budget=40960 expected_recv=32608 expected_overflow=0 exact_recv_per_rank=[32868, 32678, 32456, 32696, 32833, 33096, 32909, 32608] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=7 calls=1 budget=40960 expected_recv=32516 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=8 calls=1 budget=40960 expected_recv=32833 expected_overflow=0 exact_recv_per_rank=[32868, 32678, 32456, 32696, 32833, 33096, 32909, 32608] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=8 calls=1 budget=40960 expected_recv=32608 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=8 calls=1 budget=40960 expected_recv=32833 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=8 calls=1 budget=40960 expected_recv=32678 expected_overflow=0 exact_recv_per_rank=[32868, 32678, 32456, 32696, 32833, 33096, 32909, 32608] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=8 calls=1 budget=40960 expected_recv=32456 expected_overflow=0 exact_recv_per_rank=[32868, 32678, 32456, 32696, 32833, 33096, 32909, 32608] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32868, 32678, 32456, 32696, 32833, 33096, 32909, 32608] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=8 calls=1 budget=40960 expected_recv=32678 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=8 calls=1 budget=40960 expected_recv=32868 expected_overflow=0 exact_recv_per_rank=[32868, 32678, 32456, 32696, 32833, 33096, 32909, 32608] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=8 calls=1 budget=40960 expected_recv=32456 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=8 calls=1 budget=40960 expected_recv=32909 expected_overflow=0 exact_recv_per_rank=[32868, 32678, 32456, 32696, 32833, 33096, 32909, 32608] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=8 calls=1 budget=40960 expected_recv=32868 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=8 calls=1 budget=40960 expected_recv=32909 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=8 calls=1 budget=40960 expected_recv=32696 expected_overflow=0 exact_recv_per_rank=[32868, 32678, 32456, 32696, 32833, 33096, 32909, 32608] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=9 calls=1 budget=40960 expected_recv=32609 expected_overflow=0 exact_recv_per_rank=[33069, 32920, 32772, 32675, 32792, 32211, 33096, 32609] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=9 calls=1 budget=40960 expected_recv=32792 expected_overflow=0 exact_recv_per_rank=[33069, 32920, 32772, 32675, 32792, 32211, 33096, 32609] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=8 calls=1 budget=40960 expected_recv=32696 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=9 calls=1 budget=40960 expected_recv=32609 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=9 calls=1 budget=40960 expected_recv=32920 expected_overflow=0 exact_recv_per_rank=[33069, 32920, 32772, 32675, 32792, 32211, 33096, 32609] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=9 calls=1 budget=40960 expected_recv=32792 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=9 calls=1 budget=40960 expected_recv=32772 expected_overflow=0 exact_recv_per_rank=[33069, 32920, 32772, 32675, 32792, 32211, 33096, 32609] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=9 calls=1 budget=40960 expected_recv=32211 expected_overflow=0 exact_recv_per_rank=[33069, 32920, 32772, 32675, 32792, 32211, 33096, 32609] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=9 calls=1 budget=40960 expected_recv=32920 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=9 calls=1 budget=40960 expected_recv=32772 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=9 calls=1 budget=40960 expected_recv=32211 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=9 calls=1 budget=40960 expected_recv=33069 expected_overflow=0 exact_recv_per_rank=[33069, 32920, 32772, 32675, 32792, 32211, 33096, 32609] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33069, 32920, 32772, 32675, 32792, 32211, 33096, 32609] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=9 calls=1 budget=40960 expected_recv=33069 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=9 calls=1 budget=40960 expected_recv=32675 expected_overflow=0 exact_recv_per_rank=[33069, 32920, 32772, 32675, 32792, 32211, 33096, 32609] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=1 tokens_per_ep_rank=[32880, 32719, 32613, 32940, 32759, 32972, 32640, 32621] avg=32768.0000 max=32972 max_rank=5 min=32613 min_rank=2 imbalance_ratio=1.006226 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=9 calls=1 budget=40960 expected_recv=32675 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=1 tokens_per_ep_rank=[32880, 32719, 32613, 32940, 32759, 32972, 32640, 32621] avg=32768.0000 max=32972 max_rank=5 min=32613 min_rank=2 imbalance_ratio=1.006226 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=1 tokens_per_ep_rank=[32880, 32719, 32613, 32940, 32759, 32972, 32640, 32621] avg=32768.0000 max=32972 max_rank=5 min=32613 min_rank=2 imbalance_ratio=1.006226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=1 tokens_per_ep_rank=[32880, 32719, 32613, 32940, 32759, 32972, 32640, 32621] avg=32768.0000 max=32972 max_rank=5 min=32613 min_rank=2 imbalance_ratio=1.006226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=1 tokens_per_ep_rank=[32880, 32719, 32613, 32940, 32759, 32972, 32640, 32621] avg=32768.0000 max=32972 max_rank=5 min=32613 min_rank=2 imbalance_ratio=1.006226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=1 tokens_per_ep_rank=[32880, 32719, 32613, 32940, 32759, 32972, 32640, 32621] avg=32768.0000 max=32972 max_rank=5 min=32613 min_rank=2 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=1 tokens_per_ep_rank=[32880, 32719, 32613, 32940, 32759, 32972, 32640, 32621] avg=32768.0000 max=32972 max_rank=5 min=32613 min_rank=2 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=1 tokens_per_ep_rank=[32880, 32719, 32613, 32940, 32759, 32972, 32640, 32621] avg=32768.0000 max=32972 max_rank=5 min=32613 min_rank=2 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=1 tokens_per_ep_rank=[32880, 32719, 32613, 32940, 32759, 32972, 32640, 32621] avg=32768.0000 max=32972 max_rank=5 min=32613 min_rank=2 imbalance_ratio=1.006226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=1 tokens_per_ep_rank=[32880, 32719, 32613, 32940, 32759, 32972, 32640, 32621] avg=32768.0000 max=32972 max_rank=5 min=32613 min_rank=2 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=1 tokens_per_ep_rank=[32880, 32719, 32613, 32940, 32759, 32972, 32640, 32621] avg=32768.0000 max=32972 max_rank=5 min=32613 min_rank=2 imbalance_ratio=1.006226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=1 tokens_per_ep_rank=[32880, 32719, 32613, 32940, 32759, 32972, 32640, 32621] avg=32768.0000 max=32972 max_rank=5 min=32613 min_rank=2 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=1 tokens_per_ep_rank=[32880, 32719, 32613, 32940, 32759, 32972, 32640, 32621] avg=32768.0000 max=32972 max_rank=5 min=32613 min_rank=2 imbalance_ratio=1.006226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=2 tokens_per_ep_rank=[32953, 32778, 32810, 32891, 32779, 32776, 32435, 32722] avg=32768.0000 max=32953 max_rank=0 min=32435 min_rank=6 imbalance_ratio=1.005646 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=2 tokens_per_ep_rank=[32953, 32778, 32810, 32891, 32779, 32776, 32435, 32722] avg=32768.0000 max=32953 max_rank=0 min=32435 min_rank=6 imbalance_ratio=1.005646 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=1 tokens_per_ep_rank=[32880, 32719, 32613, 32940, 32759, 32972, 32640, 32621] avg=32768.0000 max=32972 max_rank=5 min=32613 min_rank=2 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=1 tokens_per_ep_rank=[32880, 32719, 32613, 32940, 32759, 32972, 32640, 32621] avg=32768.0000 max=32972 max_rank=5 min=32613 min_rank=2 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=2 tokens_per_ep_rank=[32953, 32778, 32810, 32891, 32779, 32776, 32435, 32722] avg=32768.0000 max=32953 max_rank=0 min=32435 min_rank=6 imbalance_ratio=1.005646 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=1 tokens_per_ep_rank=[32880, 32719, 32613, 32940, 32759, 32972, 32640, 32621] avg=32768.0000 max=32972 max_rank=5 min=32613 min_rank=2 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=2 tokens_per_ep_rank=[32953, 32778, 32810, 32891, 32779, 32776, 32435, 32722] avg=32768.0000 max=32953 max_rank=0 min=32435 min_rank=6 imbalance_ratio=1.005646 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=2 tokens_per_ep_rank=[32953, 32778, 32810, 32891, 32779, 32776, 32435, 32722] avg=32768.0000 max=32953 max_rank=0 min=32435 min_rank=6 imbalance_ratio=1.005646 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=2 tokens_per_ep_rank=[32953, 32778, 32810, 32891, 32779, 32776, 32435, 32722] avg=32768.0000 max=32953 max_rank=0 min=32435 min_rank=6 imbalance_ratio=1.005646 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=2 tokens_per_ep_rank=[32953, 32778, 32810, 32891, 32779, 32776, 32435, 32722] avg=32768.0000 max=32953 max_rank=0 min=32435 min_rank=6 imbalance_ratio=1.005646 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=2 tokens_per_ep_rank=[32953, 32778, 32810, 32891, 32779, 32776, 32435, 32722] avg=32768.0000 max=32953 max_rank=0 min=32435 min_rank=6 imbalance_ratio=1.005646 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=2 tokens_per_ep_rank=[32953, 32778, 32810, 32891, 32779, 32776, 32435, 32722] avg=32768.0000 max=32953 max_rank=0 min=32435 min_rank=6 imbalance_ratio=1.005646 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=2 tokens_per_ep_rank=[32953, 32778, 32810, 32891, 32779, 32776, 32435, 32722] avg=32768.0000 max=32953 max_rank=0 min=32435 min_rank=6 imbalance_ratio=1.005646 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=2 tokens_per_ep_rank=[32953, 32778, 32810, 32891, 32779, 32776, 32435, 32722] avg=32768.0000 max=32953 max_rank=0 min=32435 min_rank=6 imbalance_ratio=1.005646 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=2 tokens_per_ep_rank=[32953, 32778, 32810, 32891, 32779, 32776, 32435, 32722] avg=32768.0000 max=32953 max_rank=0 min=32435 min_rank=6 imbalance_ratio=1.005646 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=2 tokens_per_ep_rank=[32953, 32778, 32810, 32891, 32779, 32776, 32435, 32722] avg=32768.0000 max=32953 max_rank=0 min=32435 min_rank=6 imbalance_ratio=1.005646 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=3 tokens_per_ep_rank=[32795, 32739, 32741, 33056, 32694, 32642, 32686, 32791] avg=32768.0000 max=33056 max_rank=3 min=32642 min_rank=5 imbalance_ratio=1.008789 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=3 tokens_per_ep_rank=[32795, 32739, 32741, 33056, 32694, 32642, 32686, 32791] avg=32768.0000 max=33056 max_rank=3 min=32642 min_rank=5 imbalance_ratio=1.008789 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=3 tokens_per_ep_rank=[32795, 32739, 32741, 33056, 32694, 32642, 32686, 32791] avg=32768.0000 max=33056 max_rank=3 min=32642 min_rank=5 imbalance_ratio=1.008789 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=2 tokens_per_ep_rank=[32953, 32778, 32810, 32891, 32779, 32776, 32435, 32722] avg=32768.0000 max=32953 max_rank=0 min=32435 min_rank=6 imbalance_ratio=1.005646 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=2 tokens_per_ep_rank=[32953, 32778, 32810, 32891, 32779, 32776, 32435, 32722] avg=32768.0000 max=32953 max_rank=0 min=32435 min_rank=6 imbalance_ratio=1.005646 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=2 tokens_per_ep_rank=[32953, 32778, 32810, 32891, 32779, 32776, 32435, 32722] avg=32768.0000 max=32953 max_rank=0 min=32435 min_rank=6 imbalance_ratio=1.005646 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=3 tokens_per_ep_rank=[32795, 32739, 32741, 33056, 32694, 32642, 32686, 32791] avg=32768.0000 max=33056 max_rank=3 min=32642 min_rank=5 imbalance_ratio=1.008789 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=3 tokens_per_ep_rank=[32795, 32739, 32741, 33056, 32694, 32642, 32686, 32791] avg=32768.0000 max=33056 max_rank=3 min=32642 min_rank=5 imbalance_ratio=1.008789 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=3 tokens_per_ep_rank=[32795, 32739, 32741, 33056, 32694, 32642, 32686, 32791] avg=32768.0000 max=33056 max_rank=3 min=32642 min_rank=5 imbalance_ratio=1.008789 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=3 tokens_per_ep_rank=[32795, 32739, 32741, 33056, 32694, 32642, 32686, 32791] avg=32768.0000 max=33056 max_rank=3 min=32642 min_rank=5 imbalance_ratio=1.008789 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=3 tokens_per_ep_rank=[32795, 32739, 32741, 33056, 32694, 32642, 32686, 32791] avg=32768.0000 max=33056 max_rank=3 min=32642 min_rank=5 imbalance_ratio=1.008789 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=3 tokens_per_ep_rank=[32795, 32739, 32741, 33056, 32694, 32642, 32686, 32791] avg=32768.0000 max=33056 max_rank=3 min=32642 min_rank=5 imbalance_ratio=1.008789 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=3 tokens_per_ep_rank=[32795, 32739, 32741, 33056, 32694, 32642, 32686, 32791] avg=32768.0000 max=33056 max_rank=3 min=32642 min_rank=5 imbalance_ratio=1.008789 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=3 tokens_per_ep_rank=[32795, 32739, 32741, 33056, 32694, 32642, 32686, 32791] avg=32768.0000 max=33056 max_rank=3 min=32642 min_rank=5 imbalance_ratio=1.008789 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=4 tokens_per_ep_rank=[32934, 32560, 32645, 32565, 32904, 32463, 33058, 33015] avg=32768.0000 max=33058 max_rank=6 min=32463 min_rank=5 imbalance_ratio=1.008850 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=3 tokens_per_ep_rank=[32795, 32739, 32741, 33056, 32694, 32642, 32686, 32791] avg=32768.0000 max=33056 max_rank=3 min=32642 min_rank=5 imbalance_ratio=1.008789 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=3 tokens_per_ep_rank=[32795, 32739, 32741, 33056, 32694, 32642, 32686, 32791] avg=32768.0000 max=33056 max_rank=3 min=32642 min_rank=5 imbalance_ratio=1.008789 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=4 tokens_per_ep_rank=[32934, 32560, 32645, 32565, 32904, 32463, 33058, 33015] avg=32768.0000 max=33058 max_rank=6 min=32463 min_rank=5 imbalance_ratio=1.008850 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=4 tokens_per_ep_rank=[32934, 32560, 32645, 32565, 32904, 32463, 33058, 33015] avg=32768.0000 max=33058 max_rank=6 min=32463 min_rank=5 imbalance_ratio=1.008850 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=3 tokens_per_ep_rank=[32795, 32739, 32741, 33056, 32694, 32642, 32686, 32791] avg=32768.0000 max=33056 max_rank=3 min=32642 min_rank=5 imbalance_ratio=1.008789 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=4 tokens_per_ep_rank=[32934, 32560, 32645, 32565, 32904, 32463, 33058, 33015] avg=32768.0000 max=33058 max_rank=6 min=32463 min_rank=5 imbalance_ratio=1.008850 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=4 tokens_per_ep_rank=[32934, 32560, 32645, 32565, 32904, 32463, 33058, 33015] avg=32768.0000 max=33058 max_rank=6 min=32463 min_rank=5 imbalance_ratio=1.008850 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=3 tokens_per_ep_rank=[32795, 32739, 32741, 33056, 32694, 32642, 32686, 32791] avg=32768.0000 max=33056 max_rank=3 min=32642 min_rank=5 imbalance_ratio=1.008789 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=3 tokens_per_ep_rank=[32795, 32739, 32741, 33056, 32694, 32642, 32686, 32791] avg=32768.0000 max=33056 max_rank=3 min=32642 min_rank=5 imbalance_ratio=1.008789 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=4 tokens_per_ep_rank=[32934, 32560, 32645, 32565, 32904, 32463, 33058, 33015] avg=32768.0000 max=33058 max_rank=6 min=32463 min_rank=5 imbalance_ratio=1.008850 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=4 tokens_per_ep_rank=[32934, 32560, 32645, 32565, 32904, 32463, 33058, 33015] avg=32768.0000 max=33058 max_rank=6 min=32463 min_rank=5 imbalance_ratio=1.008850 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=4 tokens_per_ep_rank=[32934, 32560, 32645, 32565, 32904, 32463, 33058, 33015] avg=32768.0000 max=33058 max_rank=6 min=32463 min_rank=5 imbalance_ratio=1.008850 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=4 tokens_per_ep_rank=[32934, 32560, 32645, 32565, 32904, 32463, 33058, 33015] avg=32768.0000 max=33058 max_rank=6 min=32463 min_rank=5 imbalance_ratio=1.008850 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=5 tokens_per_ep_rank=[32844, 32780, 32709, 32780, 32865, 32737, 32572, 32857] avg=32768.0000 max=32865 max_rank=4 min=32572 min_rank=6 imbalance_ratio=1.002960 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=4 tokens_per_ep_rank=[32934, 32560, 32645, 32565, 32904, 32463, 33058, 33015] avg=32768.0000 max=33058 max_rank=6 min=32463 min_rank=5 imbalance_ratio=1.008850 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=4 tokens_per_ep_rank=[32934, 32560, 32645, 32565, 32904, 32463, 33058, 33015] avg=32768.0000 max=33058 max_rank=6 min=32463 min_rank=5 imbalance_ratio=1.008850 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=4 tokens_per_ep_rank=[32934, 32560, 32645, 32565, 32904, 32463, 33058, 33015] avg=32768.0000 max=33058 max_rank=6 min=32463 min_rank=5 imbalance_ratio=1.008850 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=4 tokens_per_ep_rank=[32934, 32560, 32645, 32565, 32904, 32463, 33058, 33015] avg=32768.0000 max=33058 max_rank=6 min=32463 min_rank=5 imbalance_ratio=1.008850 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=5 tokens_per_ep_rank=[32844, 32780, 32709, 32780, 32865, 32737, 32572, 32857] avg=32768.0000 max=32865 max_rank=4 min=32572 min_rank=6 imbalance_ratio=1.002960 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=5 tokens_per_ep_rank=[32844, 32780, 32709, 32780, 32865, 32737, 32572, 32857] avg=32768.0000 max=32865 max_rank=4 min=32572 min_rank=6 imbalance_ratio=1.002960 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=5 tokens_per_ep_rank=[32844, 32780, 32709, 32780, 32865, 32737, 32572, 32857] avg=32768.0000 max=32865 max_rank=4 min=32572 min_rank=6 imbalance_ratio=1.002960 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=5 tokens_per_ep_rank=[32844, 32780, 32709, 32780, 32865, 32737, 32572, 32857] avg=32768.0000 max=32865 max_rank=4 min=32572 min_rank=6 imbalance_ratio=1.002960 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=4 tokens_per_ep_rank=[32934, 32560, 32645, 32565, 32904, 32463, 33058, 33015] avg=32768.0000 max=33058 max_rank=6 min=32463 min_rank=5 imbalance_ratio=1.008850 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=4 tokens_per_ep_rank=[32934, 32560, 32645, 32565, 32904, 32463, 33058, 33015] avg=32768.0000 max=33058 max_rank=6 min=32463 min_rank=5 imbalance_ratio=1.008850 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=5 tokens_per_ep_rank=[32844, 32780, 32709, 32780, 32865, 32737, 32572, 32857] avg=32768.0000 max=32865 max_rank=4 min=32572 min_rank=6 imbalance_ratio=1.002960 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=4 tokens_per_ep_rank=[32934, 32560, 32645, 32565, 32904, 32463, 33058, 33015] avg=32768.0000 max=33058 max_rank=6 min=32463 min_rank=5 imbalance_ratio=1.008850 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=5 tokens_per_ep_rank=[32844, 32780, 32709, 32780, 32865, 32737, 32572, 32857] avg=32768.0000 max=32865 max_rank=4 min=32572 min_rank=6 imbalance_ratio=1.002960 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=5 tokens_per_ep_rank=[32844, 32780, 32709, 32780, 32865, 32737, 32572, 32857] avg=32768.0000 max=32865 max_rank=4 min=32572 min_rank=6 imbalance_ratio=1.002960 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=5 tokens_per_ep_rank=[32844, 32780, 32709, 32780, 32865, 32737, 32572, 32857] avg=32768.0000 max=32865 max_rank=4 min=32572 min_rank=6 imbalance_ratio=1.002960 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=6 tokens_per_ep_rank=[32432, 32968, 33118, 32450, 32775, 32550, 33287, 32564] avg=32768.0000 max=33287 max_rank=6 min=32432 min_rank=0 imbalance_ratio=1.015839 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=5 tokens_per_ep_rank=[32844, 32780, 32709, 32780, 32865, 32737, 32572, 32857] avg=32768.0000 max=32865 max_rank=4 min=32572 min_rank=6 imbalance_ratio=1.002960 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=5 tokens_per_ep_rank=[32844, 32780, 32709, 32780, 32865, 32737, 32572, 32857] avg=32768.0000 max=32865 max_rank=4 min=32572 min_rank=6 imbalance_ratio=1.002960 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=5 tokens_per_ep_rank=[32844, 32780, 32709, 32780, 32865, 32737, 32572, 32857] avg=32768.0000 max=32865 max_rank=4 min=32572 min_rank=6 imbalance_ratio=1.002960 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=6 tokens_per_ep_rank=[32432, 32968, 33118, 32450, 32775, 32550, 33287, 32564] avg=32768.0000 max=33287 max_rank=6 min=32432 min_rank=0 imbalance_ratio=1.015839 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=5 tokens_per_ep_rank=[32844, 32780, 32709, 32780, 32865, 32737, 32572, 32857] avg=32768.0000 max=32865 max_rank=4 min=32572 min_rank=6 imbalance_ratio=1.002960 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=6 tokens_per_ep_rank=[32432, 32968, 33118, 32450, 32775, 32550, 33287, 32564] avg=32768.0000 max=33287 max_rank=6 min=32432 min_rank=0 imbalance_ratio=1.015839 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=6 tokens_per_ep_rank=[32623, 32968, 33096, 32472, 32775, 32550, 33096, 32564] avg=32768.0000 max=33096 max_rank=2 min=32472 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=191 max_load_reduction_pct=0.005738 moved_to_replicas=213 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=6 tokens_per_ep_rank=[32432, 32968, 33118, 32450, 32775, 32550, 33287, 32564] avg=32768.0000 max=33287 max_rank=6 min=32432 min_rank=0 imbalance_ratio=1.015839 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=5 tokens_per_ep_rank=[32844, 32780, 32709, 32780, 32865, 32737, 32572, 32857] avg=32768.0000 max=32865 max_rank=4 min=32572 min_rank=6 imbalance_ratio=1.002960 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=5 tokens_per_ep_rank=[32844, 32780, 32709, 32780, 32865, 32737, 32572, 32857] avg=32768.0000 max=32865 max_rank=4 min=32572 min_rank=6 imbalance_ratio=1.002960 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=6 tokens_per_ep_rank=[32623, 32968, 33096, 32472, 32775, 32550, 33096, 32564] avg=32768.0000 max=33096 max_rank=2 min=32472 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=191 max_load_reduction_pct=0.005738 moved_to_replicas=213 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=5 tokens_per_ep_rank=[32844, 32780, 32709, 32780, 32865, 32737, 32572, 32857] avg=32768.0000 max=32865 max_rank=4 min=32572 min_rank=6 imbalance_ratio=1.002960 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=6 tokens_per_ep_rank=[32432, 32968, 33118, 32450, 32775, 32550, 33287, 32564] avg=32768.0000 max=33287 max_rank=6 min=32432 min_rank=0 imbalance_ratio=1.015839 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=6 tokens_per_ep_rank=[32623, 32968, 33096, 32472, 32775, 32550, 33096, 32564] avg=32768.0000 max=33096 max_rank=2 min=32472 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=191 max_load_reduction_pct=0.005738 moved_to_replicas=213 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=6 tokens_per_ep_rank=[32623, 32968, 33096, 32472, 32775, 32550, 33096, 32564] avg=32768.0000 max=33096 max_rank=2 min=32472 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=191 max_load_reduction_pct=0.005738 moved_to_replicas=213 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=7 tokens_per_ep_rank=[32618, 32655, 32850, 32829, 32730, 32580, 33151, 32731] avg=32768.0000 max=33151 max_rank=6 min=32580 min_rank=5 imbalance_ratio=1.011688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=6 tokens_per_ep_rank=[32623, 32968, 33096, 32472, 32775, 32550, 33096, 32564] avg=32768.0000 max=33096 max_rank=2 min=32472 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=191 max_load_reduction_pct=0.005738 moved_to_replicas=213 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=6 tokens_per_ep_rank=[32432, 32968, 33118, 32450, 32775, 32550, 33287, 32564] avg=32768.0000 max=33287 max_rank=6 min=32432 min_rank=0 imbalance_ratio=1.015839 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=7 tokens_per_ep_rank=[32618, 32655, 32850, 32829, 32730, 32580, 33151, 32731] avg=32768.0000 max=33151 max_rank=6 min=32580 min_rank=5 imbalance_ratio=1.011688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=6 tokens_per_ep_rank=[32432, 32968, 33118, 32450, 32775, 32550, 33287, 32564] avg=32768.0000 max=33287 max_rank=6 min=32432 min_rank=0 imbalance_ratio=1.015839 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=6 tokens_per_ep_rank=[32432, 32968, 33118, 32450, 32775, 32550, 33287, 32564] avg=32768.0000 max=33287 max_rank=6 min=32432 min_rank=0 imbalance_ratio=1.015839 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=7 tokens_per_ep_rank=[32618, 32655, 32850, 32829, 32730, 32635, 33096, 32731] avg=32768.0000 max=33096 max_rank=6 min=32618 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=55 max_load_reduction_pct=0.001659 moved_to_replicas=55 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=7 tokens_per_ep_rank=[32618, 32655, 32850, 32829, 32730, 32580, 33151, 32731] avg=32768.0000 max=33151 max_rank=6 min=32580 min_rank=5 imbalance_ratio=1.011688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=7 tokens_per_ep_rank=[32618, 32655, 32850, 32829, 32730, 32580, 33151, 32731] avg=32768.0000 max=33151 max_rank=6 min=32580 min_rank=5 imbalance_ratio=1.011688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=6 tokens_per_ep_rank=[32623, 32968, 33096, 32472, 32775, 32550, 33096, 32564] avg=32768.0000 max=33096 max_rank=2 min=32472 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=191 max_load_reduction_pct=0.005738 moved_to_replicas=213 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=7 tokens_per_ep_rank=[32618, 32655, 32850, 32829, 32730, 32635, 33096, 32731] avg=32768.0000 max=33096 max_rank=6 min=32618 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=55 max_load_reduction_pct=0.001659 moved_to_replicas=55 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=7 tokens_per_ep_rank=[32618, 32655, 32850, 32829, 32730, 32580, 33151, 32731] avg=32768.0000 max=33151 max_rank=6 min=32580 min_rank=5 imbalance_ratio=1.011688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=6 tokens_per_ep_rank=[32623, 32968, 33096, 32472, 32775, 32550, 33096, 32564] avg=32768.0000 max=33096 max_rank=2 min=32472 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=191 max_load_reduction_pct=0.005738 moved_to_replicas=213 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=6 tokens_per_ep_rank=[32623, 32968, 33096, 32472, 32775, 32550, 33096, 32564] avg=32768.0000 max=33096 max_rank=2 min=32472 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=191 max_load_reduction_pct=0.005738 moved_to_replicas=213 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=7 tokens_per_ep_rank=[32618, 32655, 32850, 32829, 32730, 32635, 33096, 32731] avg=32768.0000 max=33096 max_rank=6 min=32618 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=55 max_load_reduction_pct=0.001659 moved_to_replicas=55 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=7 tokens_per_ep_rank=[32618, 32655, 32850, 32829, 32730, 32635, 33096, 32731] avg=32768.0000 max=33096 max_rank=6 min=32618 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=55 max_load_reduction_pct=0.001659 moved_to_replicas=55 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=8 tokens_per_ep_rank=[32890, 33065, 33068, 32516, 32441, 32708, 32744, 32712] avg=32768.0000 max=33068 max_rank=2 min=32441 min_rank=4 imbalance_ratio=1.009155 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=7 tokens_per_ep_rank=[32618, 32655, 32850, 32829, 32730, 32635, 33096, 32731] avg=32768.0000 max=33096 max_rank=6 min=32618 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=55 max_load_reduction_pct=0.001659 moved_to_replicas=55 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=7 tokens_per_ep_rank=[32618, 32655, 32850, 32829, 32730, 32580, 33151, 32731] avg=32768.0000 max=33151 max_rank=6 min=32580 min_rank=5 imbalance_ratio=1.011688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=8 tokens_per_ep_rank=[32890, 33065, 33068, 32516, 32441, 32708, 32744, 32712] avg=32768.0000 max=33068 max_rank=2 min=32441 min_rank=4 imbalance_ratio=1.009155 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=7 tokens_per_ep_rank=[32618, 32655, 32850, 32829, 32730, 32580, 33151, 32731] avg=32768.0000 max=33151 max_rank=6 min=32580 min_rank=5 imbalance_ratio=1.011688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=7 tokens_per_ep_rank=[32618, 32655, 32850, 32829, 32730, 32580, 33151, 32731] avg=32768.0000 max=33151 max_rank=6 min=32580 min_rank=5 imbalance_ratio=1.011688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=8 tokens_per_ep_rank=[32890, 33065, 33068, 32516, 32441, 32708, 32744, 32712] avg=32768.0000 max=33068 max_rank=2 min=32441 min_rank=4 imbalance_ratio=1.009155 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=8 tokens_per_ep_rank=[32890, 33065, 33068, 32516, 32441, 32708, 32744, 32712] avg=32768.0000 max=33068 max_rank=2 min=32441 min_rank=4 imbalance_ratio=1.009155 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=8 tokens_per_ep_rank=[32890, 33065, 33068, 32516, 32441, 32708, 32744, 32712] avg=32768.0000 max=33068 max_rank=2 min=32441 min_rank=4 imbalance_ratio=1.009155 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=7 tokens_per_ep_rank=[32618, 32655, 32850, 32829, 32730, 32635, 33096, 32731] avg=32768.0000 max=33096 max_rank=6 min=32618 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=55 max_load_reduction_pct=0.001659 moved_to_replicas=55 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=8 tokens_per_ep_rank=[32890, 33065, 33068, 32516, 32441, 32708, 32744, 32712] avg=32768.0000 max=33068 max_rank=2 min=32441 min_rank=4 imbalance_ratio=1.009155 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=8 tokens_per_ep_rank=[32890, 33065, 33068, 32516, 32441, 32708, 32744, 32712] avg=32768.0000 max=33068 max_rank=2 min=32441 min_rank=4 imbalance_ratio=1.009155 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=7 tokens_per_ep_rank=[32618, 32655, 32850, 32829, 32730, 32635, 33096, 32731] avg=32768.0000 max=33096 max_rank=6 min=32618 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=55 max_load_reduction_pct=0.001659 moved_to_replicas=55 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=7 tokens_per_ep_rank=[32618, 32655, 32850, 32829, 32730, 32635, 33096, 32731] avg=32768.0000 max=33096 max_rank=6 min=32618 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=55 max_load_reduction_pct=0.001659 moved_to_replicas=55 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=8 tokens_per_ep_rank=[32890, 33065, 33068, 32516, 32441, 32708, 32744, 32712] avg=32768.0000 max=33068 max_rank=2 min=32441 min_rank=4 imbalance_ratio=1.009155 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=8 tokens_per_ep_rank=[32890, 33065, 33068, 32516, 32441, 32708, 32744, 32712] avg=32768.0000 max=33068 max_rank=2 min=32441 min_rank=4 imbalance_ratio=1.009155 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=9 tokens_per_ep_rank=[32868, 32678, 32407, 32696, 32833, 33145, 32909, 32608] avg=32768.0000 max=33145 max_rank=5 min=32407 min_rank=2 imbalance_ratio=1.011505 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=8 tokens_per_ep_rank=[32890, 33065, 33068, 32516, 32441, 32708, 32744, 32712] avg=32768.0000 max=33068 max_rank=2 min=32441 min_rank=4 imbalance_ratio=1.009155 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=8 tokens_per_ep_rank=[32890, 33065, 33068, 32516, 32441, 32708, 32744, 32712] avg=32768.0000 max=33068 max_rank=2 min=32441 min_rank=4 imbalance_ratio=1.009155 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=9 tokens_per_ep_rank=[32868, 32678, 32407, 32696, 32833, 33145, 32909, 32608] avg=32768.0000 max=33145 max_rank=5 min=32407 min_rank=2 imbalance_ratio=1.011505 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=9 tokens_per_ep_rank=[32868, 32678, 32456, 32696, 32833, 33096, 32909, 32608] avg=32768.0000 max=33096 max_rank=5 min=32456 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=49 max_load_reduction_pct=0.001478 moved_to_replicas=49 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=8 tokens_per_ep_rank=[32890, 33065, 33068, 32516, 32441, 32708, 32744, 32712] avg=32768.0000 max=33068 max_rank=2 min=32441 min_rank=4 imbalance_ratio=1.009155 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=8 tokens_per_ep_rank=[32890, 33065, 33068, 32516, 32441, 32708, 32744, 32712] avg=32768.0000 max=33068 max_rank=2 min=32441 min_rank=4 imbalance_ratio=1.009155 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=9 tokens_per_ep_rank=[32868, 32678, 32407, 32696, 32833, 33145, 32909, 32608] avg=32768.0000 max=33145 max_rank=5 min=32407 min_rank=2 imbalance_ratio=1.011505 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=9 tokens_per_ep_rank=[32868, 32678, 32407, 32696, 32833, 33145, 32909, 32608] avg=32768.0000 max=33145 max_rank=5 min=32407 min_rank=2 imbalance_ratio=1.011505 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=8 tokens_per_ep_rank=[32890, 33065, 33068, 32516, 32441, 32708, 32744, 32712] avg=32768.0000 max=33068 max_rank=2 min=32441 min_rank=4 imbalance_ratio=1.009155 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=9 tokens_per_ep_rank=[32868, 32678, 32456, 32696, 32833, 33096, 32909, 32608] avg=32768.0000 max=33096 max_rank=5 min=32456 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=49 max_load_reduction_pct=0.001478 moved_to_replicas=49 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=9 tokens_per_ep_rank=[32868, 32678, 32407, 32696, 32833, 33145, 32909, 32608] avg=32768.0000 max=33145 max_rank=5 min=32407 min_rank=2 imbalance_ratio=1.011505 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=8 tokens_per_ep_rank=[32890, 33065, 33068, 32516, 32441, 32708, 32744, 32712] avg=32768.0000 max=33068 max_rank=2 min=32441 min_rank=4 imbalance_ratio=1.009155 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=9 tokens_per_ep_rank=[32868, 32678, 32456, 32696, 32833, 33096, 32909, 32608] avg=32768.0000 max=33096 max_rank=5 min=32456 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=49 max_load_reduction_pct=0.001478 moved_to_replicas=49 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=8 tokens_per_ep_rank=[32890, 33065, 33068, 32516, 32441, 32708, 32744, 32712] avg=32768.0000 max=33068 max_rank=2 min=32441 min_rank=4 imbalance_ratio=1.009155 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=10 tokens_per_ep_rank=[33069, 32920, 32772, 32675, 32792, 32199, 33108, 32609] avg=32768.0000 max=33108 max_rank=6 min=32199 min_rank=5 imbalance_ratio=1.010376 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=9 tokens_per_ep_rank=[32868, 32678, 32456, 32696, 32833, 33096, 32909, 32608] avg=32768.0000 max=33096 max_rank=5 min=32456 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=49 max_load_reduction_pct=0.001478 moved_to_replicas=49 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=9 tokens_per_ep_rank=[32868, 32678, 32456, 32696, 32833, 33096, 32909, 32608] avg=32768.0000 max=33096 max_rank=5 min=32456 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=49 max_load_reduction_pct=0.001478 moved_to_replicas=49 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=10 tokens_per_ep_rank=[33069, 32920, 32772, 32675, 32792, 32199, 33108, 32609] avg=32768.0000 max=33108 max_rank=6 min=32199 min_rank=5 imbalance_ratio=1.010376 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=9 tokens_per_ep_rank=[32868, 32678, 32407, 32696, 32833, 33145, 32909, 32608] avg=32768.0000 max=33145 max_rank=5 min=32407 min_rank=2 imbalance_ratio=1.011505 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=10 tokens_per_ep_rank=[33069, 32920, 32772, 32675, 32792, 32211, 33096, 32609] avg=32768.0000 max=33096 max_rank=6 min=32211 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12 max_load_reduction_pct=0.000362 moved_to_replicas=12 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=10 tokens_per_ep_rank=[33069, 32920, 32772, 32675, 32792, 32199, 33108, 32609] avg=32768.0000 max=33108 max_rank=6 min=32199 min_rank=5 imbalance_ratio=1.010376 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=9 tokens_per_ep_rank=[32868, 32678, 32407, 32696, 32833, 33145, 32909, 32608] avg=32768.0000 max=33145 max_rank=5 min=32407 min_rank=2 imbalance_ratio=1.011505 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=9 tokens_per_ep_rank=[32868, 32678, 32407, 32696, 32833, 33145, 32909, 32608] avg=32768.0000 max=33145 max_rank=5 min=32407 min_rank=2 imbalance_ratio=1.011505 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=10 tokens_per_ep_rank=[33069, 32920, 32772, 32675, 32792, 32199, 33108, 32609] avg=32768.0000 max=33108 max_rank=6 min=32199 min_rank=5 imbalance_ratio=1.010376 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=10 tokens_per_ep_rank=[33069, 32920, 32772, 32675, 32792, 32211, 33096, 32609] avg=32768.0000 max=33096 max_rank=6 min=32211 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12 max_load_reduction_pct=0.000362 moved_to_replicas=12 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=9 tokens_per_ep_rank=[32868, 32678, 32456, 32696, 32833, 33096, 32909, 32608] avg=32768.0000 max=33096 max_rank=5 min=32456 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=49 max_load_reduction_pct=0.001478 moved_to_replicas=49 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=10 tokens_per_ep_rank=[33069, 32920, 32772, 32675, 32792, 32199, 33108, 32609] avg=32768.0000 max=33108 max_rank=6 min=32199 min_rank=5 imbalance_ratio=1.010376 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=10 tokens_per_ep_rank=[33069, 32920, 32772, 32675, 32792, 32211, 33096, 32609] avg=32768.0000 max=33096 max_rank=6 min=32211 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12 max_load_reduction_pct=0.000362 moved_to_replicas=12 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=9 tokens_per_ep_rank=[32868, 32678, 32456, 32696, 32833, 33096, 32909, 32608] avg=32768.0000 max=33096 max_rank=5 min=32456 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=49 max_load_reduction_pct=0.001478 moved_to_replicas=49 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=9 tokens_per_ep_rank=[32868, 32678, 32456, 32696, 32833, 33096, 32909, 32608] avg=32768.0000 max=33096 max_rank=5 min=32456 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=49 max_load_reduction_pct=0.001478 moved_to_replicas=49 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=10 tokens_per_ep_rank=[33069, 32920, 32772, 32675, 32792, 32211, 33096, 32609] avg=32768.0000 max=33096 max_rank=6 min=32211 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12 max_load_reduction_pct=0.000362 moved_to_replicas=12 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=10 tokens_per_ep_rank=[33069, 32920, 32772, 32675, 32792, 32211, 33096, 32609] avg=32768.0000 max=33096 max_rank=6 min=32211 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12 max_load_reduction_pct=0.000362 moved_to_replicas=12 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=10 tokens_per_ep_rank=[33069, 32920, 32772, 32675, 32792, 32199, 33108, 32609] avg=32768.0000 max=33108 max_rank=6 min=32199 min_rank=5 imbalance_ratio=1.010376 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=10 tokens_per_ep_rank=[33069, 32920, 32772, 32675, 32792, 32199, 33108, 32609] avg=32768.0000 max=33108 max_rank=6 min=32199 min_rank=5 imbalance_ratio=1.010376 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=10 tokens_per_ep_rank=[33069, 32920, 32772, 32675, 32792, 32199, 33108, 32609] avg=32768.0000 max=33108 max_rank=6 min=32199 min_rank=5 imbalance_ratio=1.010376 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=10 tokens_per_ep_rank=[33069, 32920, 32772, 32675, 32792, 32211, 33096, 32609] avg=32768.0000 max=33096 max_rank=6 min=32211 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12 max_load_reduction_pct=0.000362 moved_to_replicas=12 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=10 tokens_per_ep_rank=[33069, 32920, 32772, 32675, 32792, 32211, 33096, 32609] avg=32768.0000 max=33096 max_rank=6 min=32211 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12 max_load_reduction_pct=0.000362 moved_to_replicas=12 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=10 tokens_per_ep_rank=[33069, 32920, 32772, 32675, 32792, 32211, 33096, 32609] avg=32768.0000 max=33096 max_rank=6 min=32211 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12 max_load_reduction_pct=0.000362 moved_to_replicas=12 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=11 records=10 total_expected_recv=327230 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=77824 peak_demand=33015 required_capacity=1.007538 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=11 records=10 total_expected_recv=327572 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76928 peak_demand=32904 required_capacity=1.004150 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=11 records=10 total_expected_recv=326790 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76800 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=11 records=10 total_expected_recv=327862 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76160 peak_demand=33065 required_capacity=1.009064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=11 records=10 total_expected_recv=327760 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76544 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=11 records=10 total_expected_recv=328474 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76416 peak_demand=33069 required_capacity=1.009186 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=11 records=10 total_expected_recv=327420 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76800 peak_demand=33056 required_capacity=1.008789 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=11 records=10 total_expected_recv=328332 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76672 peak_demand=33096 required_capacity=1.010010 + [2026-08-26 15:53:14] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 215.9 | throughput per GPU (TFLOP/s/GPU): 572.0 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.451500E+00 | loss scale: 1.0 | grad norm: 3.486 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 33664 | min safe capacity: 1.0273 | +[after training is done] datetime: 2026-08-26 15:53:14 +[rank0]:[W826 15:53:15.631710130 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W0826 15:53:16.223000 149908 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0826 15:53:16.223000 149908 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 149998 closing signal SIGTERM +W0826 15:53:16.224000 149908 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 149999 closing signal SIGTERM +W0826 15:53:16.226000 149908 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 150000 closing signal SIGTERM +W0826 15:53:16.227000 149908 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 150001 closing signal SIGTERM +W0826 15:53:16.228000 149908 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 150002 closing signal SIGTERM +W0826 15:53:16.230000 149908 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 150003 closing signal SIGTERM +W0826 15:53:16.231000 149908 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 150004 closing signal SIGTERM +W0826 15:53:16.232000 149908 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 150005 closing signal SIGTERM +Traceback (most recent call last): + File "/usr/local/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run + time.sleep(monitor_interval) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 149908 got signal: 15 diff --git a/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/capacity_attempts.tsv b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/capacity_attempts.tsv index 54cd1c0796ff4c43de5a9e36de6ef12123878348..82e7388e7e22c0835ebb7b9fc46452f50d83566b 100644 --- a/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/capacity_attempts.tsv +++ b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/capacity_attempts.tsv @@ -6,3 +6,4 @@ QWEN3_30B ultraep forced 1.25 OK 117867 1 0.0 1.01001 /workspace/tierb_study_202 DSV2_LITE_L24 ultraep measure 1.25 OK 130023 1 0.0 1.010071 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25/rep1 DSV2_LITE_L24 ultraep forced 1.25 OK 129971 1 0.0 1.01001 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__forcebal/rep1 QWEN2_57B ultraep measure 1.25 OK 97675 1 0.0 1.010101 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p25/rep1 +QWEN2_57B ultraep forced 1.25 OK 97677 1 0.0 1.01001 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p25__forcebal/rep1 diff --git a/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/capacity_selection.tsv b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/capacity_selection.tsv index 39eea8c931163982b6059f4e4997398a4c5112da..934ec5b5c76d5ecbcdf2eab4f728dba19f2969af 100644 --- a/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/capacity_selection.tsv +++ b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/capacity_selection.tsv @@ -2,3 +2,4 @@ model backend status selected_capacity required_capacity drop_rate_pct peak_mib QWEN3_235B ultraep SELECTED 1.25 1.010056 0.0 122751 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25/rep1 QWEN3_30B ultraep SELECTED 1.25 1.010101 0.0 117927 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25/rep1 DSV2_LITE_L24 ultraep SELECTED 1.25 1.010071 0.0 130023 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25/rep1 +QWEN2_57B ultraep SELECTED 1.25 1.010101 0.0 97675 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p25/rep1 diff --git a/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/probe_summary.json b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/probe_summary.json index 7df05f5558f3b7aecfa9f1687654d71d266c4a29..28240fed82a8e8b2098ef15dadccd1c836c901d0 100644 --- a/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/probe_summary.json +++ b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/probe_summary.json @@ -76,6 +76,237 @@ "valid_max_rank_reduction_n": 276, "zero_drop": 1 }, + { + "arm": "forced", + "assignment_sum_mismatches": 0, + "aux_loss_coeff": "1e-3", + "backend": "ultraep", + "branch": "ultraep-token-drop-analysis", + "capacity": "1.25", + "capacity_mode": "static", + "commit": "eaf829e7", + "config": "DSV2_LITE_L24", + "cuda_graph": "0", + "dense_first_layers": "1", + "drop_expected_assignments": 108527616.0, + "drop_rate_pct_max": 0.0, + "drop_rate_pct_mean": 0.0, + "drop_rate_pct_min": 0.0, + "drop_rate_pct_n": 96, + "drop_rate_pct_overall": 0.0, + "dropped_tokens_sum_all_records": 0.0, + "duplicate_vector_mismatches": 0, + "experts_per_rank": "8", + "force_balance": "1", + "global_batch_size": "8", + "global_snapshots": 276, + "incomplete_snapshots": 0, + "malformed_records": 0, + "moe_layers": "23", + "num_experts": "64", + "num_layers": "24", + "observed_dropped_tokens": 0.0, + "peak_demand_max": 49644.0, + "peak_demand_mean": 49527.270833333336, + "peak_demand_min": 49296.0, + "peak_demand_n": 96, + "peak_rank_overflow_max": 0.0, + "peak_rank_overflow_mean": 0.0, + "peak_rank_overflow_min": 0.0, + "peak_rank_overflow_n": 96, + "post_physical_rank_cv_max": 0.006673533732490181, + "post_physical_rank_cv_mean": 0.003906455227664182, + "post_physical_rank_cv_min": 0.0014708009315538319, + "post_physical_rank_cv_n": 276, + "post_physical_rank_imax_max": 1.010009765625, + "post_physical_rank_imax_mean": 1.0060408587617, + "post_physical_rank_imax_min": 1.00213623046875, + "post_physical_rank_imax_n": 276, + "pre_home_rank_cv_max": 0.006715970721309454, + "pre_home_rank_cv_mean": 0.00392157619549587, + "pre_home_rank_cv_min": 0.0014708009315538319, + "pre_home_rank_cv_n": 276, + "pre_home_rank_imax_max": 1.0122884114583333, + "pre_home_rank_imax_mean": 1.0060781580238527, + "pre_home_rank_imax_min": 1.00213623046875, + "pre_home_rank_imax_n": 276, + "records": { + "drop_dispatch": 2208, + "drop_experts": 2208, + "drop_setup": 2208, + "ultraep_drop": 96, + "ultraep_post": 2208, + "ultraep_pre": 2208 + }, + "required_capacity_max": 1.01001, + "required_capacity_mean": 1.007635, + "required_capacity_min": 1.00293, + "required_capacity_n": 96, + "router_balancing": "seq_aux_loss", + "run_dir": "DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__forcebal/rep1", + "seq_length": "8192", + "shared_expert_overlap": "0", + "topk": "6", + "valid_max_rank_reduction_max": 0.0022509848058525606, + "valid_max_rank_reduction_mean": 3.688638533861224e-05, + "valid_max_rank_reduction_min": 0.0, + "valid_max_rank_reduction_n": 276, + "zero_drop": 1 + }, + { + "arm": "natural", + "assignment_sum_mismatches": 0, + "aux_loss_coeff": "1e-2", + "backend": "ultraep", + "branch": "ultraep-token-drop-analysis", + "capacity": "1.25", + "capacity_mode": "static", + "commit": "eaf829e7", + "config": "QWEN2_57B", + "cuda_graph": "0", + "dense_first_layers": "0", + "drop_expected_assignments": 31457280.0, + "drop_rate_pct_max": 0.0, + "drop_rate_pct_mean": 0.0, + "drop_rate_pct_min": 0.0, + "drop_rate_pct_n": 96, + "drop_rate_pct_overall": 0.0, + "dropped_tokens_sum_all_records": 0.0, + "duplicate_vector_mismatches": 0, + "experts_per_rank": "8", + "force_balance": "0", + "global_batch_size": "8", + "global_snapshots": 120, + "incomplete_snapshots": 0, + "malformed_records": 0, + "moe_layers": "10", + "num_experts": "64", + "num_layers": "10", + "observed_dropped_tokens": 0.0, + "peak_demand_max": 33099.0, + "peak_demand_mean": 33096.770833333336, + "peak_demand_min": 33096.0, + "peak_demand_n": 96, + "peak_rank_overflow_max": 0.0, + "peak_rank_overflow_mean": 0.0, + "peak_rank_overflow_min": 0.0, + "peak_rank_overflow_n": 96, + "post_physical_rank_cv_max": 0.026483363713168166, + "post_physical_rank_cv_mean": 0.016169459913295655, + "post_physical_rank_cv_min": 0.004221172387878395, + "post_physical_rank_cv_n": 120, + "post_physical_rank_imax_max": 1.010101318359375, + "post_physical_rank_imax_mean": 1.0100074768066407, + "post_physical_rank_imax_min": 1.00701904296875, + "post_physical_rank_imax_n": 120, + "pre_home_rank_cv_max": 1.4028263272834383, + "pre_home_rank_cv_mean": 0.6344045914264498, + "pre_home_rank_cv_min": 0.004221172387878395, + "pre_home_rank_cv_n": 120, + "pre_home_rank_imax_max": 4.42657470703125, + "pre_home_rank_imax_mean": 2.1723561604817707, + "pre_home_rank_imax_min": 1.00701904296875, + "pre_home_rank_imax_n": 120, + "records": { + "drop_dispatch": 960, + "drop_experts": 960, + "drop_setup": 960, + "ultraep_drop": 96, + "ultraep_post": 960, + "ultraep_pre": 960 + }, + "required_capacity_max": 1.010101, + "required_capacity_mean": 1.0100332395833334, + "required_capacity_min": 1.01001, + "required_capacity_n": 96, + "router_balancing": "aux_loss", + "run_dir": "QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p25/rep1", + "seq_length": "4096", + "shared_expert_overlap": "0", + "topk": "8", + "valid_max_rank_reduction_max": 0.7718235091347811, + "valid_max_rank_reduction_mean": 0.49113710577671815, + "valid_max_rank_reduction_min": 0.0, + "valid_max_rank_reduction_n": 120, + "zero_drop": 1 + }, + { + "arm": "forced", + "assignment_sum_mismatches": 0, + "aux_loss_coeff": "1e-2", + "backend": "ultraep", + "branch": "ultraep-token-drop-analysis", + "capacity": "1.25", + "capacity_mode": "static", + "commit": "eaf829e7", + "config": "QWEN2_57B", + "cuda_graph": "0", + "dense_first_layers": "0", + "drop_expected_assignments": 31457280.0, + "drop_rate_pct_max": 0.0, + "drop_rate_pct_mean": 0.0, + "drop_rate_pct_min": 0.0, + "drop_rate_pct_n": 96, + "drop_rate_pct_overall": 0.0, + "dropped_tokens_sum_all_records": 0.0, + "duplicate_vector_mismatches": 0, + "experts_per_rank": "8", + "force_balance": "1", + "global_batch_size": "8", + "global_snapshots": 120, + "incomplete_snapshots": 0, + "malformed_records": 0, + "moe_layers": "10", + "num_experts": "64", + "num_layers": "10", + "observed_dropped_tokens": 0.0, + "peak_demand_max": 33096.0, + "peak_demand_mean": 33004.5625, + "peak_demand_min": 32800.0, + "peak_demand_n": 96, + "peak_rank_overflow_max": 0.0, + "peak_rank_overflow_mean": 0.0, + "peak_rank_overflow_min": 0.0, + "peak_rank_overflow_n": 96, + "post_physical_rank_cv_max": 0.01039906345549811, + "post_physical_rank_cv_mean": 0.004650204419961747, + "post_physical_rank_cv_min": 0.002294812301292466, + "post_physical_rank_cv_n": 120, + "post_physical_rank_imax_max": 1.010009765625, + "post_physical_rank_imax_mean": 1.0071016947428386, + "post_physical_rank_imax_min": 1.002899169921875, + "post_physical_rank_imax_n": 120, + "pre_home_rank_cv_max": 0.01039906345549811, + "pre_home_rank_cv_mean": 0.004760489966476766, + "pre_home_rank_cv_min": 0.002294812301292466, + "pre_home_rank_cv_n": 120, + "pre_home_rank_imax_max": 1.015838623046875, + "pre_home_rank_imax_mean": 1.0073875427246093, + "pre_home_rank_imax_min": 1.002899169921875, + "pre_home_rank_imax_n": 120, + "records": { + "drop_dispatch": 960, + "drop_experts": 960, + "drop_setup": 960, + "ultraep_drop": 96, + "ultraep_post": 960, + "ultraep_pre": 960 + }, + "required_capacity_max": 1.01001, + "required_capacity_mean": 1.0072193541666667, + "required_capacity_min": 1.000977, + "required_capacity_n": 96, + "router_balancing": "aux_loss", + "run_dir": "QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p25__forcebal/rep1", + "seq_length": "4096", + "shared_expert_overlap": "0", + "topk": "8", + "valid_max_rank_reduction_max": 0.005737975786343017, + "valid_max_rank_reduction_mean": 0.00028222750974011727, + "valid_max_rank_reduction_min": 0.0, + "valid_max_rank_reduction_n": 120, + "zero_drop": 1 + }, { "arm": "natural", "assignment_sum_mismatches": 0, @@ -106,8 +337,8 @@ "num_experts": "128", "num_layers": "12", "observed_dropped_tokens": 0.0, - "peak_demand_max": 66194.0, - "peak_demand_mean": 66192.60416666667, + "peak_demand_max": 66195.0, + "peak_demand_mean": 66192.71875, "peak_demand_min": 66192.0, "peak_demand_n": 96, "peak_rank_overflow_max": 0.0, @@ -115,19 +346,19 @@ "peak_rank_overflow_min": 0.0, "peak_rank_overflow_n": 96, "post_physical_rank_cv_max": 0.026489118276835043, - "post_physical_rank_cv_mean": 0.014698204478704438, - "post_physical_rank_cv_min": 0.004171709396704442, + "post_physical_rank_cv_mean": 0.014509436266048916, + "post_physical_rank_cv_min": 0.004304689378357216, "post_physical_rank_cv_n": 144, - "post_physical_rank_imax_max": 1.010040283203125, - "post_physical_rank_imax_mean": 1.0100169711642795, + "post_physical_rank_imax_max": 1.0100555419921875, + "post_physical_rank_imax_mean": 1.010018030802409, "post_physical_rank_imax_min": 1.010009765625, "post_physical_rank_imax_n": 144, - "pre_home_rank_cv_max": 1.3850730653619154, - "pre_home_rank_cv_mean": 0.8051429963522313, + "pre_home_rank_cv_max": 1.4793990428288122, + "pre_home_rank_cv_mean": 0.7888747618345466, "pre_home_rank_cv_min": 0.3167748701963148, "pre_home_rank_cv_n": 144, - "pre_home_rank_imax_max": 4.0391845703125, - "pre_home_rank_imax_mean": 2.465598636203342, + "pre_home_rank_imax_max": 4.008636474609375, + "pre_home_rank_imax_mean": 2.4475171830919056, "pre_home_rank_imax_min": 1.2317352294921875, "pre_home_rank_imax_n": 144, "records": { @@ -138,8 +369,8 @@ "ultraep_post": 1152, "ultraep_pre": 1152 }, - "required_capacity_max": 1.01004, - "required_capacity_mean": 1.0100190625, + "required_capacity_max": 1.010056, + "required_capacity_mean": 1.0100207916666666, "required_capacity_min": 1.01001, "required_capacity_n": 96, "router_balancing": "aux_loss", @@ -147,12 +378,89 @@ "seq_length": "8192", "shared_expert_overlap": "0", "topk": "8", - "valid_max_rank_reduction_max": 0.7499471123334038, - "valid_max_rank_reduction_mean": 0.5612683072031022, + "valid_max_rank_reduction_max": 0.7480415667466027, + "valid_max_rank_reduction_mean": 0.5583849946461037, "valid_max_rank_reduction_min": 0.17998587763091065, "valid_max_rank_reduction_n": 144, "zero_drop": 1 }, + { + "arm": "forced", + "assignment_sum_mismatches": 0, + "aux_loss_coeff": "1e-3", + "backend": "ultraep", + "branch": "ultraep-token-drop-analysis", + "capacity": "1.25", + "capacity_mode": "static", + "commit": "eaf829e7", + "config": "QWEN3_235B", + "cuda_graph": "0", + "dense_first_layers": "0", + "drop_expected_assignments": 75497472.0, + "drop_rate_pct_max": 0.0, + "drop_rate_pct_mean": 0.0, + "drop_rate_pct_min": 0.0, + "drop_rate_pct_n": 96, + "drop_rate_pct_overall": 0.0, + "dropped_tokens_sum_all_records": 0.0, + "duplicate_vector_mismatches": 0, + "experts_per_rank": "16", + "force_balance": "1", + "global_batch_size": "8", + "global_snapshots": 144, + "incomplete_snapshots": 0, + "malformed_records": 0, + "moe_layers": "12", + "num_experts": "128", + "num_layers": "12", + "observed_dropped_tokens": 0.0, + "peak_demand_max": 66192.0, + "peak_demand_mean": 65914.44791666667, + "peak_demand_min": 65589.0, + "peak_demand_n": 96, + "peak_rank_overflow_max": 0.0, + "peak_rank_overflow_mean": 0.0, + "peak_rank_overflow_min": 0.0, + "peak_rank_overflow_n": 96, + "post_physical_rank_cv_max": 0.006838559601482533, + "post_physical_rank_cv_mean": 0.003410368479189458, + "post_physical_rank_cv_min": 0.0012782366924169522, + "post_physical_rank_cv_n": 144, + "post_physical_rank_imax_max": 1.010009765625, + "post_physical_rank_imax_mean": 1.0053078333536785, + "post_physical_rank_imax_min": 1.0019378662109375, + "post_physical_rank_imax_n": 144, + "pre_home_rank_cv_max": 0.006990045273933762, + "pre_home_rank_cv_mean": 0.0034121784262079076, + "pre_home_rank_cv_min": 0.0012782366924169522, + "pre_home_rank_cv_n": 144, + "pre_home_rank_imax_max": 1.0104522705078125, + "pre_home_rank_imax_mean": 1.0053127076890733, + "pre_home_rank_imax_min": 1.0019378662109375, + "pre_home_rank_imax_n": 144, + "records": { + "drop_dispatch": 1152, + "drop_experts": 1152, + "drop_setup": 1152, + "ultraep_drop": 96, + "ultraep_post": 1152, + "ultraep_pre": 1152 + }, + "required_capacity_max": 1.01001, + "required_capacity_mean": 1.0057746770833333, + "required_capacity_min": 1.000809, + "required_capacity_n": 96, + "router_balancing": "aux_loss", + "run_dir": "QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__forcebal/rep1", + "seq_length": "8192", + "shared_expert_overlap": "0", + "topk": "8", + "valid_max_rank_reduction_max": 0.00043792754564262094, + "valid_max_rank_reduction_mean": 4.82445946182061e-06, + "valid_max_rank_reduction_min": 0.0, + "valid_max_rank_reduction_n": 144, + "zero_drop": 1 + }, { "arm": "natural", "assignment_sum_mismatches": 0, @@ -184,28 +492,28 @@ "num_layers": "48", "observed_dropped_tokens": 0.0, "peak_demand_max": 33099.0, - "peak_demand_mean": 33097.229166666664, + "peak_demand_mean": 33097.208333333336, "peak_demand_min": 33096.0, "peak_demand_n": 96, "peak_rank_overflow_max": 0.0, "peak_rank_overflow_mean": 0.0, "peak_rank_overflow_min": 0.0, "peak_rank_overflow_n": 96, - "post_physical_rank_cv_max": 0.02649488696921753, - "post_physical_rank_cv_mean": 0.013615268745845822, - "post_physical_rank_cv_min": 0.0044932923824075426, + "post_physical_rank_cv_max": 0.026517962359754763, + "post_physical_rank_cv_mean": 0.01354621389621944, + "post_physical_rank_cv_min": 0.00409507351694325, "post_physical_rank_cv_n": 576, "post_physical_rank_imax_max": 1.010101318359375, - "post_physical_rank_imax_mean": 1.0100238058302138, - "post_physical_rank_imax_min": 1.00994873046875, + "post_physical_rank_imax_mean": 1.0100230111016169, + "post_physical_rank_imax_min": 1.009979248046875, "post_physical_rank_imax_n": 576, - "pre_home_rank_cv_max": 1.577101326466163, - "pre_home_rank_cv_mean": 0.7366559250184537, - "pre_home_rank_cv_min": 0.14094636542440406, + "pre_home_rank_cv_max": 1.5733118377040045, + "pre_home_rank_cv_mean": 0.7595764660737883, + "pre_home_rank_cv_min": 0.010628488683156438, "pre_home_rank_cv_n": 576, - "pre_home_rank_imax_max": 4.9957275390625, - "pre_home_rank_imax_mean": 2.2919526629977756, - "pre_home_rank_imax_min": 1.2030029296875, + "pre_home_rank_imax_max": 4.98095703125, + "pre_home_rank_imax_mean": 2.3515779177347818, + "pre_home_rank_imax_min": 1.02227783203125, "pre_home_rank_imax_n": 576, "records": { "drop_dispatch": 4608, @@ -216,7 +524,7 @@ "ultraep_pre": 4608 }, "required_capacity_max": 1.010101, - "required_capacity_mean": 1.0100472291666667, + "required_capacity_mean": 1.0100465833333334, "required_capacity_min": 1.01001, "required_capacity_n": 96, "router_balancing": "aux_loss", @@ -224,9 +532,86 @@ "seq_length": "4096", "shared_expert_overlap": "0", "topk": "8", - "valid_max_rank_reduction_max": 0.7978252901649359, - "valid_max_rank_reduction_mean": 0.5268142670026216, - "valid_max_rank_reduction_min": 0.1604261796042618, + "valid_max_rank_reduction_max": 0.7972257621801784, + "valid_max_rank_reduction_mean": 0.5366741569155073, + "valid_max_rank_reduction_min": 0.01200071646068422, + "valid_max_rank_reduction_n": 576, + "zero_drop": 1 + }, + { + "arm": "forced", + "assignment_sum_mismatches": 0, + "aux_loss_coeff": "1e-3", + "backend": "ultraep", + "branch": "ultraep-token-drop-analysis", + "capacity": "1.25", + "capacity_mode": "static", + "commit": "eaf829e7", + "config": "QWEN3_30B", + "cuda_graph": "0", + "dense_first_layers": "0", + "drop_expected_assignments": 150994944.0, + "drop_rate_pct_max": 0.0, + "drop_rate_pct_mean": 0.0, + "drop_rate_pct_min": 0.0, + "drop_rate_pct_n": 96, + "drop_rate_pct_overall": 0.0, + "dropped_tokens_sum_all_records": 0.0, + "duplicate_vector_mismatches": 0, + "experts_per_rank": "16", + "force_balance": "1", + "global_batch_size": "8", + "global_snapshots": 576, + "incomplete_snapshots": 0, + "malformed_records": 0, + "moe_layers": "48", + "num_experts": "128", + "num_layers": "48", + "observed_dropped_tokens": 0.0, + "peak_demand_max": 33096.0, + "peak_demand_mean": 33084.114583333336, + "peak_demand_min": 32979.0, + "peak_demand_n": 96, + "peak_rank_overflow_max": 0.0, + "peak_rank_overflow_mean": 0.0, + "peak_rank_overflow_min": 0.0, + "peak_rank_overflow_n": 96, + "post_physical_rank_cv_max": 0.008082507253598464, + "post_physical_rank_cv_mean": 0.004779118222581316, + "post_physical_rank_cv_min": 0.0012089160228135533, + "post_physical_rank_cv_n": 576, + "post_physical_rank_imax_max": 1.010009765625, + "post_physical_rank_imax_mean": 1.0073255962795682, + "post_physical_rank_imax_min": 1.00177001953125, + "post_physical_rank_imax_n": 576, + "pre_home_rank_cv_max": 0.009112333689945906, + "pre_home_rank_cv_mean": 0.004932308997139817, + "pre_home_rank_cv_min": 0.0012089160228135533, + "pre_home_rank_cv_n": 576, + "pre_home_rank_imax_max": 1.01739501953125, + "pre_home_rank_imax_mean": 1.0077196756998699, + "pre_home_rank_imax_min": 1.00177001953125, + "pre_home_rank_imax_n": 576, + "records": { + "drop_dispatch": 4608, + "drop_experts": 4608, + "drop_setup": 4608, + "ultraep_drop": 96, + "ultraep_post": 4608, + "ultraep_pre": 4608 + }, + "required_capacity_max": 1.01001, + "required_capacity_mean": 1.0096472395833334, + "required_capacity_min": 1.006439, + "required_capacity_n": 96, + "router_balancing": "aux_loss", + "run_dir": "QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__forcebal/rep1", + "seq_length": "4096", + "shared_expert_overlap": "0", + "topk": "8", + "valid_max_rank_reduction_max": 0.007258983742276081, + "valid_max_rank_reduction_mean": 0.00038872034774766455, + "valid_max_rank_reduction_min": 0.0, "valid_max_rank_reduction_n": 576, "zero_drop": 1 } diff --git a/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/probe_summary.tsv b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/probe_summary.tsv index 7a90f8017942c8e50cda3fc5e415727980e08049..8e96951e7ec0985f36c6b2b44f9ada07f1fd297c 100644 --- a/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/probe_summary.tsv +++ b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/probe_summary.tsv @@ -1,4 +1,9 @@ run_dir arm config backend num_experts topk experts_per_rank num_layers moe_layers dense_first_layers seq_length global_batch_size capacity_mode capacity force_balance cuda_graph branch commit router_balancing aux_loss_coeff shared_expert_overlap assignment_sum_mismatches drop_expected_assignments drop_rate_pct_max drop_rate_pct_mean drop_rate_pct_min drop_rate_pct_n drop_rate_pct_overall dropped_tokens_sum_all_records duplicate_vector_mismatches global_snapshots incomplete_snapshots malformed_records observed_dropped_tokens peak_demand_max peak_demand_mean peak_demand_min peak_demand_n peak_rank_overflow_max peak_rank_overflow_mean peak_rank_overflow_min peak_rank_overflow_n post_physical_rank_cv_max post_physical_rank_cv_mean post_physical_rank_cv_min post_physical_rank_cv_n post_physical_rank_imax_max post_physical_rank_imax_mean post_physical_rank_imax_min post_physical_rank_imax_n pre_home_rank_cv_max pre_home_rank_cv_mean pre_home_rank_cv_min pre_home_rank_cv_n pre_home_rank_imax_max pre_home_rank_imax_mean pre_home_rank_imax_min pre_home_rank_imax_n required_capacity_max required_capacity_mean required_capacity_min required_capacity_n valid_max_rank_reduction_max valid_max_rank_reduction_mean valid_max_rank_reduction_min valid_max_rank_reduction_n zero_drop DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25/rep1 natural DSV2_LITE_L24 ultraep 64 6 8 24 23 1 8192 8 static 1.25 0 0 ultraep-token-drop-analysis eaf829e7 seq_aux_loss 1e-3 0 0 108527616.0 0.0 0.0 0.0 96 0.0 0.0 0 276 0 0 0.0 49647.0 49645.270833333336 49644.0 96 0.0 0.0 0.0 96 0.02649873914445656 0.020751556297178456 0.010393449619293449 276 1.01007080078125 1.0100281941142057 1.010009765625 276 1.3897787536248405 0.5881957299328056 0.06749209466238901 276 4.4163818359375 2.107403446510794 1.1109822591145833 276 1.010071 1.0100354270833332 1.01001 96 0.7712991882952357 0.46558143481151487 0.09086747120332558 276 1 -QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25/rep1 natural QWEN3_235B ultraep 128 8 16 12 12 0 8192 8 static 1.25 0 0 ultraep-token-drop-analysis eaf829e7 aux_loss 1e-3 0 0 75497472.0 0.0 0.0 0.0 96 0.0 0.0 0 144 0 0 0.0 66194.0 66192.60416666667 66192.0 96 0.0 0.0 0.0 96 0.026489118276835043 0.014698204478704438 0.004171709396704442 144 1.010040283203125 1.0100169711642795 1.010009765625 144 1.3850730653619154 0.8051429963522313 0.3167748701963148 144 4.0391845703125 2.465598636203342 1.2317352294921875 144 1.01004 1.0100190625 1.01001 96 0.7499471123334038 0.5612683072031022 0.17998587763091065 144 1 -QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25/rep1 natural QWEN3_30B ultraep 128 8 16 48 48 0 4096 8 static 1.25 0 0 ultraep-token-drop-analysis eaf829e7 aux_loss 1e-3 0 0 150994944.0 0.0 0.0 0.0 96 0.0 0.0 0 576 0 0 0.0 33099.0 33097.229166666664 33096.0 96 0.0 0.0 0.0 96 0.02649488696921753 0.013615268745845822 0.0044932923824075426 576 1.010101318359375 1.0100238058302138 1.00994873046875 576 1.577101326466163 0.7366559250184537 0.14094636542440406 576 4.9957275390625 2.2919526629977756 1.2030029296875 576 1.010101 1.0100472291666667 1.01001 96 0.7978252901649359 0.5268142670026216 0.1604261796042618 576 1 +DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__forcebal/rep1 forced DSV2_LITE_L24 ultraep 64 6 8 24 23 1 8192 8 static 1.25 1 0 ultraep-token-drop-analysis eaf829e7 seq_aux_loss 1e-3 0 0 108527616.0 0.0 0.0 0.0 96 0.0 0.0 0 276 0 0 0.0 49644.0 49527.270833333336 49296.0 96 0.0 0.0 0.0 96 0.006673533732490181 0.003906455227664182 0.0014708009315538319 276 1.010009765625 1.0060408587617 1.00213623046875 276 0.006715970721309454 0.00392157619549587 0.0014708009315538319 276 1.0122884114583333 1.0060781580238527 1.00213623046875 276 1.01001 1.007635 1.00293 96 0.0022509848058525606 3.688638533861224e-05 0.0 276 1 +QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p25/rep1 natural QWEN2_57B ultraep 64 8 8 10 10 0 4096 8 static 1.25 0 0 ultraep-token-drop-analysis eaf829e7 aux_loss 1e-2 0 0 31457280.0 0.0 0.0 0.0 96 0.0 0.0 0 120 0 0 0.0 33099.0 33096.770833333336 33096.0 96 0.0 0.0 0.0 96 0.026483363713168166 0.016169459913295655 0.004221172387878395 120 1.010101318359375 1.0100074768066407 1.00701904296875 120 1.4028263272834383 0.6344045914264498 0.004221172387878395 120 4.42657470703125 2.1723561604817707 1.00701904296875 120 1.010101 1.0100332395833334 1.01001 96 0.7718235091347811 0.49113710577671815 0.0 120 1 +QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p25__forcebal/rep1 forced QWEN2_57B ultraep 64 8 8 10 10 0 4096 8 static 1.25 1 0 ultraep-token-drop-analysis eaf829e7 aux_loss 1e-2 0 0 31457280.0 0.0 0.0 0.0 96 0.0 0.0 0 120 0 0 0.0 33096.0 33004.5625 32800.0 96 0.0 0.0 0.0 96 0.01039906345549811 0.004650204419961747 0.002294812301292466 120 1.010009765625 1.0071016947428386 1.002899169921875 120 0.01039906345549811 0.004760489966476766 0.002294812301292466 120 1.015838623046875 1.0073875427246093 1.002899169921875 120 1.01001 1.0072193541666667 1.000977 96 0.005737975786343017 0.00028222750974011727 0.0 120 1 +QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25/rep1 natural QWEN3_235B ultraep 128 8 16 12 12 0 8192 8 static 1.25 0 0 ultraep-token-drop-analysis eaf829e7 aux_loss 1e-3 0 0 75497472.0 0.0 0.0 0.0 96 0.0 0.0 0 144 0 0 0.0 66195.0 66192.71875 66192.0 96 0.0 0.0 0.0 96 0.026489118276835043 0.014509436266048916 0.004304689378357216 144 1.0100555419921875 1.010018030802409 1.010009765625 144 1.4793990428288122 0.7888747618345466 0.3167748701963148 144 4.008636474609375 2.4475171830919056 1.2317352294921875 144 1.010056 1.0100207916666666 1.01001 96 0.7480415667466027 0.5583849946461037 0.17998587763091065 144 1 +QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__forcebal/rep1 forced QWEN3_235B ultraep 128 8 16 12 12 0 8192 8 static 1.25 1 0 ultraep-token-drop-analysis eaf829e7 aux_loss 1e-3 0 0 75497472.0 0.0 0.0 0.0 96 0.0 0.0 0 144 0 0 0.0 66192.0 65914.44791666667 65589.0 96 0.0 0.0 0.0 96 0.006838559601482533 0.003410368479189458 0.0012782366924169522 144 1.010009765625 1.0053078333536785 1.0019378662109375 144 0.006990045273933762 0.0034121784262079076 0.0012782366924169522 144 1.0104522705078125 1.0053127076890733 1.0019378662109375 144 1.01001 1.0057746770833333 1.000809 96 0.00043792754564262094 4.82445946182061e-06 0.0 144 1 +QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25/rep1 natural QWEN3_30B ultraep 128 8 16 48 48 0 4096 8 static 1.25 0 0 ultraep-token-drop-analysis eaf829e7 aux_loss 1e-3 0 0 150994944.0 0.0 0.0 0.0 96 0.0 0.0 0 576 0 0 0.0 33099.0 33097.208333333336 33096.0 96 0.0 0.0 0.0 96 0.026517962359754763 0.01354621389621944 0.00409507351694325 576 1.010101318359375 1.0100230111016169 1.009979248046875 576 1.5733118377040045 0.7595764660737883 0.010628488683156438 576 4.98095703125 2.3515779177347818 1.02227783203125 576 1.010101 1.0100465833333334 1.01001 96 0.7972257621801784 0.5366741569155073 0.01200071646068422 576 1 +QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__forcebal/rep1 forced QWEN3_30B ultraep 128 8 16 48 48 0 4096 8 static 1.25 1 0 ultraep-token-drop-analysis eaf829e7 aux_loss 1e-3 0 0 150994944.0 0.0 0.0 0.0 96 0.0 0.0 0 576 0 0 0.0 33096.0 33084.114583333336 32979.0 96 0.0 0.0 0.0 96 0.008082507253598464 0.004779118222581316 0.0012089160228135533 576 1.010009765625 1.0073255962795682 1.00177001953125 576 0.009112333689945906 0.004932308997139817 0.0012089160228135533 576 1.01739501953125 1.0077196756998699 1.00177001953125 576 1.01001 1.0096472395833334 1.006439 96 0.007258983742276081 0.00038872034774766455 0.0 576 1 diff --git a/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/results.tsv b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/results.tsv index add80251e873ecdef765198db59bd1235186bea4..226eb28f53035c11614168c6e07a9d84560c2b36 100644 --- a/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/results.tsv +++ b/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/results.tsv @@ -6,3 +6,4 @@ QWEN3_30B ultraep matched 16 4096 8 1.25 none+forcebal 1 OK 737.56 705.60 117867 DSV2_LITE_L24 ultraep matched 8 8192 8 1.25 none 1 OK 639.96 638.80 130023 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25/rep1 DSV2_LITE_L24 ultraep matched 8 8192 8 1.25 none+forcebal 1 OK 625.83 624.60 129971 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__forcebal/rep1 QWEN2_57B ultraep matched 8 4096 8 1.25 none 1 OK 238.29 230.10 97675 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p25/rep1 +QWEN2_57B ultraep matched 8 4096 8 1.25 none+forcebal 1 OK 269.69 212.00 97677 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p25__forcebal/rep1